services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/sirupsen/logrus"
  31         check "gopkg.in/check.v1"
  32 )
  33
  34 var _ = check.Suite(&UnitSuite{})
  35
  36 func init() {
  37         arvados.DebugLocksPanicMode = true
  38 }
  39
  40 type UnitSuite struct {
  41         Config *arvados.Config
  42 }
  43
  44 func (s *UnitSuite) SetUpTest(c *check.C) {
  45         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  46         ldr.Path = "-"
  47         cfg, err := ldr.Load()
  48         c.Assert(err, check.IsNil)
  49         s.Config = cfg
  50 }
  51
  52 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  53         h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)}
  54         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  55         req := &http.Request{
  56                 Method:     "OPTIONS",
  57                 Host:       u.Host,
  58                 URL:        u,
  59                 RequestURI: u.RequestURI(),
  60                 Header: http.Header{
  61                         "Origin":                        {"https://workbench.example"},
  62                         "Access-Control-Request-Method": {"POST"},
  63                 },
  64         }
  65
  66         // Check preflight for an allowed request
  67         resp := httptest.NewRecorder()
  68         h.ServeHTTP(resp, req)
  69         c.Check(resp.Code, check.Equals, http.StatusOK)
  70         c.Check(resp.Body.String(), check.Equals, "")
  71         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  72         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  73         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  74
  75         // Check preflight for a disallowed request
  76         resp = httptest.NewRecorder()
  77         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  78         h.ServeHTTP(resp, req)
  79         c.Check(resp.Body.String(), check.Equals, "")
  80         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  81 }
  82
  83 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
  84         for _, trial := range []struct {
  85                 dataExists    bool
  86                 sendIMSHeader bool
  87                 expectStatus  int
  88                 logRegexp     string
  89         }{
  90                 // If we return no content due to a Keep read error,
  91                 // we should emit a log message.
  92                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
  93
  94                 // If we return no content because the client sent an
  95                 // If-Modified-Since header, our response should be
  96                 // 304.  We still expect a "File download" log since it
  97                 // counts as a file access for auditing.
  98                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
  99         } {
 100                 c.Logf("trial: %+v", trial)
 101                 arvadostest.StartKeep(2, true)
 102                 if trial.dataExists {
 103                         arv, err := arvadosclient.MakeArvadosClient()
 104                         c.Assert(err, check.IsNil)
 105                         arv.ApiToken = arvadostest.ActiveToken
 106                         kc, err := keepclient.MakeKeepClient(arv)
 107                         c.Assert(err, check.IsNil)
 108                         _, _, err = kc.PutB([]byte("foo"))
 109                         c.Assert(err, check.IsNil)
 110                 }
 111
 112                 h := handler{Config: newConfig(ctxlog.TestLogger(c), s.Config)}
 113                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 114                 req := &http.Request{
 115                         Method:     "GET",
 116                         Host:       u.Host,
 117                         URL:        u,
 118                         RequestURI: u.RequestURI(),
 119                         Header: http.Header{
 120                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 121                         },
 122                 }
 123                 if trial.sendIMSHeader {
 124                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 125                 }
 126
 127                 var logbuf bytes.Buffer
 128                 logger := logrus.New()
 129                 logger.Out = &logbuf
 130                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 131
 132                 resp := httptest.NewRecorder()
 133                 h.ServeHTTP(resp, req)
 134                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 135                 c.Check(resp.Body.String(), check.Equals, "")
 136
 137                 c.Log(logbuf.String())
 138                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 139         }
 140 }
 141
 142 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 143         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 144         token := arvadostest.ActiveToken
 145         for _, trial := range []string{
 146                 "http://keep-web/c=" + bogusID + "/foo",
 147                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 148                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 149                 "http://keep-web/collections/" + bogusID + "/foo",
 150                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 151                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 152         } {
 153                 c.Log(trial)
 154                 u := mustParseURL(trial)
 155                 req := &http.Request{
 156                         Method:     "GET",
 157                         Host:       u.Host,
 158                         URL:        u,
 159                         RequestURI: u.RequestURI(),
 160                 }
 161                 resp := httptest.NewRecorder()
 162                 cfg := newConfig(ctxlog.TestLogger(c), s.Config)
 163                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 164                 h := handler{Config: cfg}
 165                 h.ServeHTTP(resp, req)
 166                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 167         }
 168 }
 169
 170 func mustParseURL(s string) *url.URL {
 171         r, err := url.Parse(s)
 172         if err != nil {
 173                 panic("parse URL: " + s)
 174         }
 175         return r
 176 }
 177
 178 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 179         for _, testURL := range []string{
 180                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 181                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 182         } {
 183                 resp := httptest.NewRecorder()
 184                 u := mustParseURL(testURL)
 185                 req := &http.Request{
 186                         Method:     "GET",
 187                         URL:        u,
 188                         RequestURI: u.RequestURI(),
 189                 }
 190                 s.testServer.Handler.ServeHTTP(resp, req)
 191                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 192                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 193         }
 194 }
 195
 196 // An authorizer modifies an HTTP request to make use of the given
 197 // token -- by adding it to a header, cookie, query param, or whatever
 198 // -- and returns the HTTP status code we should expect from keep-web if
 199 // the token is invalid.
 200 type authorizer func(*http.Request, string) int
 201
 202 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 203         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 204 }
 205 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 206         r.Header.Add("Authorization", "Bearer "+tok)
 207         return http.StatusUnauthorized
 208 }
 209 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 210         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 211 }
 212 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 213         r.Header.Add("Authorization", "Bearer "+tok)
 214         return http.StatusUnauthorized
 215 }
 216
 217 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 218         s.doVhostRequests(c, authzViaCookieValue)
 219 }
 220 func authzViaCookieValue(r *http.Request, tok string) int {
 221         r.AddCookie(&http.Cookie{
 222                 Name:  "arvados_api_token",
 223                 Value: auth.EncodeTokenCookie([]byte(tok)),
 224         })
 225         return http.StatusUnauthorized
 226 }
 227
 228 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 229         s.doVhostRequests(c, authzViaPath)
 230 }
 231 func authzViaPath(r *http.Request, tok string) int {
 232         r.URL.Path = "/t=" + tok + r.URL.Path
 233         return http.StatusNotFound
 234 }
 235
 236 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 237         s.doVhostRequests(c, authzViaQueryString)
 238 }
 239 func authzViaQueryString(r *http.Request, tok string) int {
 240         r.URL.RawQuery = "api_token=" + tok
 241         return http.StatusUnauthorized
 242 }
 243
 244 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 245         s.doVhostRequests(c, authzViaPOST)
 246 }
 247 func authzViaPOST(r *http.Request, tok string) int {
 248         r.Method = "POST"
 249         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 250         r.Body = ioutil.NopCloser(strings.NewReader(
 251                 url.Values{"api_token": {tok}}.Encode()))
 252         return http.StatusUnauthorized
 253 }
 254
 255 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 256         s.doVhostRequests(c, authzViaPOST)
 257 }
 258 func authzViaXHRPOST(r *http.Request, tok string) int {
 259         r.Method = "POST"
 260         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 261         r.Header.Add("Origin", "https://origin.example")
 262         r.Body = ioutil.NopCloser(strings.NewReader(
 263                 url.Values{
 264                         "api_token":   {tok},
 265                         "disposition": {"attachment"},
 266                 }.Encode()))
 267         return http.StatusUnauthorized
 268 }
 269
 270 // Try some combinations of {url, token} using the given authorization
 271 // mechanism, and verify the result is correct.
 272 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 273         for _, hostPath := range []string{
 274                 arvadostest.FooCollection + ".example.com/foo",
 275                 arvadostest.FooCollection + "--collections.example.com/foo",
 276                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 277                 arvadostest.FooCollectionPDH + ".example.com/foo",
 278                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 279                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 280         } {
 281                 c.Log("doRequests: ", hostPath)
 282                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 283         }
 284 }
 285
 286 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 287         for _, tok := range []string{
 288                 arvadostest.ActiveToken,
 289                 arvadostest.ActiveToken[:15],
 290                 arvadostest.SpectatorToken,
 291                 "bogus",
 292                 "",
 293         } {
 294                 u := mustParseURL("http://" + hostPath)
 295                 req := &http.Request{
 296                         Method:     "GET",
 297                         Host:       u.Host,
 298                         URL:        u,
 299                         RequestURI: u.RequestURI(),
 300                         Header:     http.Header{},
 301                 }
 302                 failCode := authz(req, tok)
 303                 req, resp := s.doReq(req)
 304                 code, body := resp.Code, resp.Body.String()
 305
 306                 // If the initial request had a (non-empty) token
 307                 // showing in the query string, we should have been
 308                 // redirected in order to hide it in a cookie.
 309                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 310
 311                 if tok == arvadostest.ActiveToken {
 312                         c.Check(code, check.Equals, http.StatusOK)
 313                         c.Check(body, check.Equals, "foo")
 314                 } else {
 315                         c.Check(code >= 400, check.Equals, true)
 316                         c.Check(code < 500, check.Equals, true)
 317                         if tok == arvadostest.SpectatorToken {
 318                                 // Valid token never offers to retry
 319                                 // with different credentials.
 320                                 c.Check(code, check.Equals, http.StatusNotFound)
 321                         } else {
 322                                 // Invalid token can ask to retry
 323                                 // depending on the authz method.
 324                                 c.Check(code, check.Equals, failCode)
 325                         }
 326                         if code == 404 {
 327                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 328                         } else {
 329                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 330                         }
 331                 }
 332         }
 333 }
 334
 335 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 336         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 337                 for _, port := range []string{"80", "443", "8000"} {
 338                         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 339                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 340                         req := &http.Request{
 341                                 Method:     "GET",
 342                                 Host:       u.Host,
 343                                 URL:        u,
 344                                 RequestURI: u.RequestURI(),
 345                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 346                         }
 347                         req, resp := s.doReq(req)
 348                         code, _ := resp.Code, resp.Body.String()
 349
 350                         if port == "8000" {
 351                                 c.Check(code, check.Equals, 401)
 352                         } else {
 353                                 c.Check(code, check.Equals, 200)
 354                         }
 355                 }
 356         }
 357 }
 358
 359 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 360         resp := httptest.NewRecorder()
 361         s.testServer.Handler.ServeHTTP(resp, req)
 362         if resp.Code != http.StatusSeeOther {
 363                 return req, resp
 364         }
 365         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 366         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 367         req = &http.Request{
 368                 Method:     "GET",
 369                 Host:       u.Host,
 370                 URL:        u,
 371                 RequestURI: u.RequestURI(),
 372                 Header:     http.Header{},
 373         }
 374         for _, c := range cookies {
 375                 req.AddCookie(c)
 376         }
 377         return s.doReq(req)
 378 }
 379
 380 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 381         s.testVhostRedirectTokenToCookie(c, "GET",
 382                 arvadostest.FooCollection+".example.com/foo",
 383                 "?api_token="+arvadostest.ActiveToken,
 384                 "",
 385                 "",
 386                 http.StatusOK,
 387                 "foo",
 388         )
 389 }
 390
 391 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 392         s.testVhostRedirectTokenToCookie(c, "GET",
 393                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 394                 "",
 395                 "",
 396                 "",
 397                 http.StatusOK,
 398                 "foo",
 399         )
 400 }
 401
 402 // Bad token in URL is 404 Not Found because it doesn't make sense to
 403 // retry the same URL with different authorization.
 404 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 405         s.testVhostRedirectTokenToCookie(c, "GET",
 406                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 407                 "",
 408                 "",
 409                 "",
 410                 http.StatusNotFound,
 411                 notFoundMessage+"\n",
 412         )
 413 }
 414
 415 // Bad token in a cookie (even if it got there via our own
 416 // query-string-to-cookie redirect) is, in principle, retryable at the
 417 // same URL so it's 401 Unauthorized.
 418 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 419         s.testVhostRedirectTokenToCookie(c, "GET",
 420                 arvadostest.FooCollection+".example.com/foo",
 421                 "?api_token=thisisabogustoken",
 422                 "",
 423                 "",
 424                 http.StatusUnauthorized,
 425                 unauthorizedMessage+"\n",
 426         )
 427 }
 428
 429 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 430         s.testVhostRedirectTokenToCookie(c, "GET",
 431                 "example.com/c="+arvadostest.FooCollection+"/foo",
 432                 "?api_token="+arvadostest.ActiveToken,
 433                 "",
 434                 "",
 435                 http.StatusBadRequest,
 436                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 437         )
 438 }
 439
 440 // If client requests an attachment by putting ?disposition=attachment
 441 // in the query string, and gets redirected, the redirect target
 442 // should respond with an attachment.
 443 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 444         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 445                 arvadostest.FooCollection+".example.com/foo",
 446                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 447                 "",
 448                 "",
 449                 http.StatusOK,
 450                 "foo",
 451         )
 452         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 453 }
 454
 455 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 456         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 457         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 458                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 459                 "?api_token="+arvadostest.ActiveToken,
 460                 "",
 461                 "",
 462                 http.StatusOK,
 463                 "foo",
 464         )
 465         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 466 }
 467
 468 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 469         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 470         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 471                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 472                 "?api_token="+arvadostest.ActiveToken,
 473                 "",
 474                 "",
 475                 http.StatusOK,
 476                 "waz",
 477         )
 478         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 479         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 480                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 481                 "?api_token="+arvadostest.ActiveToken,
 482                 "",
 483                 "",
 484                 http.StatusOK,
 485                 "waz",
 486         )
 487         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 488 }
 489
 490 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 491         s.testServer.Config.cluster.Collections.TrustAllContent = true
 492         s.testVhostRedirectTokenToCookie(c, "GET",
 493                 "example.com/c="+arvadostest.FooCollection+"/foo",
 494                 "?api_token="+arvadostest.ActiveToken,
 495                 "",
 496                 "",
 497                 http.StatusOK,
 498                 "foo",
 499         )
 500 }
 501
 502 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 503         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 504
 505         s.testVhostRedirectTokenToCookie(c, "GET",
 506                 "example.com/c="+arvadostest.FooCollection+"/foo",
 507                 "?api_token="+arvadostest.ActiveToken,
 508                 "",
 509                 "",
 510                 http.StatusBadRequest,
 511                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 512         )
 513
 514         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 515                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 516                 "?api_token="+arvadostest.ActiveToken,
 517                 "",
 518                 "",
 519                 http.StatusOK,
 520                 "foo",
 521         )
 522         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 523 }
 524
 525 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 526         s.testVhostRedirectTokenToCookie(c, "POST",
 527                 arvadostest.FooCollection+".example.com/foo",
 528                 "",
 529                 "application/x-www-form-urlencoded",
 530                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 531                 http.StatusOK,
 532                 "foo",
 533         )
 534 }
 535
 536 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 537         s.testVhostRedirectTokenToCookie(c, "POST",
 538                 arvadostest.FooCollection+".example.com/foo",
 539                 "",
 540                 "application/x-www-form-urlencoded",
 541                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 542                 http.StatusNotFound,
 543                 notFoundMessage+"\n",
 544         )
 545 }
 546
 547 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 548         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 549         s.testVhostRedirectTokenToCookie(c, "GET",
 550                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 551                 "",
 552                 "",
 553                 "",
 554                 http.StatusOK,
 555                 "Hello world\n",
 556         )
 557 }
 558
 559 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 560         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 561         s.testVhostRedirectTokenToCookie(c, "GET",
 562                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 563                 "",
 564                 "",
 565                 "",
 566                 http.StatusNotFound,
 567                 notFoundMessage+"\n",
 568         )
 569 }
 570
 571 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 572         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 573
 574         client := s.testServer.Config.Client
 575         client.AuthToken = arvadostest.ActiveToken
 576         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 577         c.Assert(err, check.IsNil)
 578         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 579         c.Assert(err, check.IsNil)
 580         f.Close()
 581         mtxt, err := fs.MarshalManifest(".")
 582         c.Assert(err, check.IsNil)
 583         var coll arvados.Collection
 584         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 585                 "collection": map[string]string{
 586                         "manifest_text": mtxt,
 587                 },
 588         })
 589         c.Assert(err, check.IsNil)
 590
 591         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 592         req := &http.Request{
 593                 Method:     "GET",
 594                 Host:       u.Host,
 595                 URL:        u,
 596                 RequestURI: u.RequestURI(),
 597                 Header: http.Header{
 598                         "Authorization": {"Bearer " + client.AuthToken},
 599                 },
 600         }
 601         resp := httptest.NewRecorder()
 602         s.testServer.Handler.ServeHTTP(resp, req)
 603         c.Check(resp.Code, check.Equals, http.StatusOK)
 604         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 605 }
 606
 607 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 608         arv := arvados.NewClientFromEnv()
 609         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 610         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 611         name := "foo/bar/baz"
 612         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 613         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 614
 615         client := s.testServer.Config.Client
 616         client.AuthToken = arvadostest.ActiveToken
 617         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 618         c.Assert(err, check.IsNil)
 619         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 620         c.Assert(err, check.IsNil)
 621         f.Close()
 622         mtxt, err := fs.MarshalManifest(".")
 623         c.Assert(err, check.IsNil)
 624         var coll arvados.Collection
 625         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 626                 "collection": map[string]string{
 627                         "manifest_text": mtxt,
 628                         "name":          name,
 629                         "owner_uuid":    arvadostest.AProjectUUID,
 630                 },
 631         })
 632         c.Assert(err, check.IsNil)
 633         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 634
 635         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 636         for tryURL, expectRegexp := range map[string]string{
 637                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 638                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 639         } {
 640                 u, _ := url.Parse(tryURL)
 641                 req := &http.Request{
 642                         Method:     "GET",
 643                         Host:       u.Host,
 644                         URL:        u,
 645                         RequestURI: u.RequestURI(),
 646                         Header: http.Header{
 647                                 "Authorization": {"Bearer " + client.AuthToken},
 648                         },
 649                 }
 650                 resp := httptest.NewRecorder()
 651                 s.testServer.Handler.ServeHTTP(resp, req)
 652                 c.Check(resp.Code, check.Equals, http.StatusOK)
 653                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 654         }
 655 }
 656
 657 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 658 // and disposition=attachment (telling us it's acceptable to respond
 659 // with content instead of a redirect) and an Origin header that gets
 660 // added automatically by the browser (telling us it's desirable to do
 661 // so).
 662 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 663         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 664         req := &http.Request{
 665                 Method:     "POST",
 666                 Host:       u.Host,
 667                 URL:        u,
 668                 RequestURI: u.RequestURI(),
 669                 Header: http.Header{
 670                         "Origin":       {"https://origin.example"},
 671                         "Content-Type": {"application/x-www-form-urlencoded"},
 672                 },
 673                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 674                         "api_token":   {arvadostest.ActiveToken},
 675                         "disposition": {"attachment"},
 676                 }.Encode())),
 677         }
 678         resp := httptest.NewRecorder()
 679         s.testServer.Handler.ServeHTTP(resp, req)
 680         c.Check(resp.Code, check.Equals, http.StatusOK)
 681         c.Check(resp.Body.String(), check.Equals, "foo")
 682         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 683
 684         // GET + Origin header is representative of both AJAX GET
 685         // requests and inline images via <IMG crossorigin="anonymous"
 686         // src="...">.
 687         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 688         req = &http.Request{
 689                 Method:     "GET",
 690                 Host:       u.Host,
 691                 URL:        u,
 692                 RequestURI: u.RequestURI(),
 693                 Header: http.Header{
 694                         "Origin": {"https://origin.example"},
 695                 },
 696         }
 697         resp = httptest.NewRecorder()
 698         s.testServer.Handler.ServeHTTP(resp, req)
 699         c.Check(resp.Code, check.Equals, http.StatusOK)
 700         c.Check(resp.Body.String(), check.Equals, "foo")
 701         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 702 }
 703
 704 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 705         u, _ := url.Parse(`http://` + hostPath + queryString)
 706         req := &http.Request{
 707                 Method:     method,
 708                 Host:       u.Host,
 709                 URL:        u,
 710                 RequestURI: u.RequestURI(),
 711                 Header:     http.Header{"Content-Type": {contentType}},
 712                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 713         }
 714
 715         resp := httptest.NewRecorder()
 716         defer func() {
 717                 c.Check(resp.Code, check.Equals, expectStatus)
 718                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 719         }()
 720
 721         s.testServer.Handler.ServeHTTP(resp, req)
 722         if resp.Code != http.StatusSeeOther {
 723                 return resp
 724         }
 725         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 726         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 727
 728         u, _ = u.Parse(resp.Header().Get("Location"))
 729         req = &http.Request{
 730                 Method:     "GET",
 731                 Host:       u.Host,
 732                 URL:        u,
 733                 RequestURI: u.RequestURI(),
 734                 Header:     http.Header{},
 735         }
 736         for _, c := range cookies {
 737                 req.AddCookie(c)
 738         }
 739
 740         resp = httptest.NewRecorder()
 741         s.testServer.Handler.ServeHTTP(resp, req)
 742         c.Check(resp.Header().Get("Location"), check.Equals, "")
 743         return resp
 744 }
 745
 746 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 747         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 748         s.testDirectoryListing(c)
 749 }
 750
 751 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 752         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 753         s.testDirectoryListing(c)
 754 }
 755
 756 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 757         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 758         authHeader := http.Header{
 759                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 760         }
 761         for _, trial := range []struct {
 762                 uri      string
 763                 header   http.Header
 764                 expect   []string
 765                 redirect string
 766                 cutDirs  int
 767         }{
 768                 {
 769                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 770                         header:  authHeader,
 771                         expect:  []string{"dir1/foo", "dir1/bar"},
 772                         cutDirs: 0,
 773                 },
 774                 {
 775                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 776                         header:  authHeader,
 777                         expect:  []string{"foo", "bar"},
 778                         cutDirs: 1,
 779                 },
 780                 {
 781                         // URLs of this form ignore authHeader, and
 782                         // FooAndBarFilesInDirUUID isn't public, so
 783                         // this returns 401.
 784                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 785                         header: authHeader,
 786                         expect: nil,
 787                 },
 788                 {
 789                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 790                         header:  authHeader,
 791                         expect:  []string{"dir1/"},
 792                         cutDirs: 3,
 793                 },
 794                 {
 795                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 796                         header:  authHeader,
 797                         expect:  []string{"bar"},
 798                         cutDirs: 4,
 799                 },
 800                 {
 801                         uri:     "download.example.com/",
 802                         header:  authHeader,
 803                         expect:  []string{"users/"},
 804                         cutDirs: 0,
 805                 },
 806                 {
 807                         uri:      "download.example.com/users",
 808                         header:   authHeader,
 809                         redirect: "/users/",
 810                         expect:   []string{"active/"},
 811                         cutDirs:  1,
 812                 },
 813                 {
 814                         uri:     "download.example.com/users/",
 815                         header:  authHeader,
 816                         expect:  []string{"active/"},
 817                         cutDirs: 1,
 818                 },
 819                 {
 820                         uri:      "download.example.com/users/active",
 821                         header:   authHeader,
 822                         redirect: "/users/active/",
 823                         expect:   []string{"foo_file_in_dir/"},
 824                         cutDirs:  2,
 825                 },
 826                 {
 827                         uri:     "download.example.com/users/active/",
 828                         header:  authHeader,
 829                         expect:  []string{"foo_file_in_dir/"},
 830                         cutDirs: 2,
 831                 },
 832                 {
 833                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 834                         header:  nil,
 835                         expect:  []string{"dir1/foo", "dir1/bar"},
 836                         cutDirs: 4,
 837                 },
 838                 {
 839                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 840                         header:  nil,
 841                         expect:  []string{"dir1/foo", "dir1/bar"},
 842                         cutDirs: 2,
 843                 },
 844                 {
 845                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 846                         header:  nil,
 847                         expect:  []string{"dir1/foo", "dir1/bar"},
 848                         cutDirs: 2,
 849                 },
 850                 {
 851                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 852                         header:  authHeader,
 853                         expect:  []string{"dir1/foo", "dir1/bar"},
 854                         cutDirs: 1,
 855                 },
 856                 {
 857                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 858                         header:   authHeader,
 859                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 860                         expect:   []string{"foo", "bar"},
 861                         cutDirs:  2,
 862                 },
 863                 {
 864                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 865                         header:  authHeader,
 866                         expect:  []string{"foo", "bar"},
 867                         cutDirs: 3,
 868                 },
 869                 {
 870                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 871                         header:   authHeader,
 872                         redirect: "/dir1/",
 873                         expect:   []string{"foo", "bar"},
 874                         cutDirs:  1,
 875                 },
 876                 {
 877                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 878                         header: authHeader,
 879                         expect: nil,
 880                 },
 881                 {
 882                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 883                         header:  authHeader,
 884                         expect:  []string{"waz"},
 885                         cutDirs: 1,
 886                 },
 887                 {
 888                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 889                         header:  authHeader,
 890                         expect:  []string{"waz"},
 891                         cutDirs: 2,
 892                 },
 893         } {
 894                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 895                 resp := httptest.NewRecorder()
 896                 u := mustParseURL("//" + trial.uri)
 897                 req := &http.Request{
 898                         Method:     "GET",
 899                         Host:       u.Host,
 900                         URL:        u,
 901                         RequestURI: u.RequestURI(),
 902                         Header:     copyHeader(trial.header),
 903                 }
 904                 s.testServer.Handler.ServeHTTP(resp, req)
 905                 var cookies []*http.Cookie
 906                 for resp.Code == http.StatusSeeOther {
 907                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 908                         req = &http.Request{
 909                                 Method:     "GET",
 910                                 Host:       u.Host,
 911                                 URL:        u,
 912                                 RequestURI: u.RequestURI(),
 913                                 Header:     copyHeader(trial.header),
 914                         }
 915                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 916                         for _, c := range cookies {
 917                                 req.AddCookie(c)
 918                         }
 919                         resp = httptest.NewRecorder()
 920                         s.testServer.Handler.ServeHTTP(resp, req)
 921                 }
 922                 if trial.redirect != "" {
 923                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 924                 }
 925                 if trial.expect == nil {
 926                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 927                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 928                         } else {
 929                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 930                         }
 931                 } else {
 932                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 933                         for _, e := range trial.expect {
 934                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 935                         }
 936                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 937                 }
 938
 939                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 940                 req = &http.Request{
 941                         Method:     "OPTIONS",
 942                         Host:       u.Host,
 943                         URL:        u,
 944                         RequestURI: u.RequestURI(),
 945                         Header:     copyHeader(trial.header),
 946                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 947                 }
 948                 resp = httptest.NewRecorder()
 949                 s.testServer.Handler.ServeHTTP(resp, req)
 950                 if trial.expect == nil {
 951                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 952                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 953                         } else {
 954                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 955                         }
 956                 } else {
 957                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 958                 }
 959
 960                 req = &http.Request{
 961                         Method:     "PROPFIND",
 962                         Host:       u.Host,
 963                         URL:        u,
 964                         RequestURI: u.RequestURI(),
 965                         Header:     copyHeader(trial.header),
 966                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 967                 }
 968                 resp = httptest.NewRecorder()
 969                 s.testServer.Handler.ServeHTTP(resp, req)
 970                 if trial.expect == nil {
 971                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 972                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 973                         } else {
 974                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 975                         }
 976                 } else {
 977                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 978                         for _, e := range trial.expect {
 979                                 if strings.HasSuffix(e, "/") {
 980                                         e = filepath.Join(u.Path, e) + "/"
 981                                 } else {
 982                                         e = filepath.Join(u.Path, e)
 983                                 }
 984                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 985                         }
 986                 }
 987         }
 988 }
 989
 990 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 991         arv := arvados.NewClientFromEnv()
 992         var newCollection arvados.Collection
 993         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 994                 "collection": map[string]string{
 995                         "owner_uuid":    arvadostest.ActiveUserUUID,
 996                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 997                         "name":          "keep-web test collection",
 998                 },
 999                 "ensure_unique_name": true,
1000         })
1001         c.Assert(err, check.IsNil)
1002         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1003
1004         var updated arvados.Collection
1005         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1006                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1007                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1008                 req := &http.Request{
1009                         Method:     "DELETE",
1010                         Host:       u.Host,
1011                         URL:        u,
1012                         RequestURI: u.RequestURI(),
1013                         Header: http.Header{
1014                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1015                         },
1016                 }
1017                 resp := httptest.NewRecorder()
1018                 s.testServer.Handler.ServeHTTP(resp, req)
1019                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1020
1021                 updated = arvados.Collection{}
1022                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1023                 c.Check(err, check.IsNil)
1024                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1025                 c.Logf("updated manifest_text %q", updated.ManifestText)
1026         }
1027         c.Check(updated.ManifestText, check.Equals, "")
1028 }
1029
1030 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
1031         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
1032         authHeader := http.Header{
1033                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
1034         }
1035
1036         resp := httptest.NewRecorder()
1037         u := mustParseURL("http://download.example.com/_health/ping")
1038         req := &http.Request{
1039                 Method:     "GET",
1040                 Host:       u.Host,
1041                 URL:        u,
1042                 RequestURI: u.RequestURI(),
1043                 Header:     authHeader,
1044         }
1045         s.testServer.Handler.ServeHTTP(resp, req)
1046
1047         c.Check(resp.Code, check.Equals, http.StatusOK)
1048         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
1049 }
1050
1051 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1052         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1053
1054         client := s.testServer.Config.Client
1055         client.AuthToken = arvadostest.ActiveToken
1056         arv, err := arvadosclient.New(&client)
1057         c.Assert(err, check.Equals, nil)
1058         kc, err := keepclient.MakeKeepClient(arv)
1059         c.Assert(err, check.Equals, nil)
1060
1061         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
1062         c.Assert(err, check.IsNil)
1063
1064         trials := []struct {
1065                 filename    string
1066                 content     string
1067                 contentType string
1068         }{
1069                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1070                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1071                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1072                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1073                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1074         }
1075         for _, trial := range trials {
1076                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1077                 c.Assert(err, check.IsNil)
1078                 _, err = f.Write([]byte(trial.content))
1079                 c.Assert(err, check.IsNil)
1080                 c.Assert(f.Close(), check.IsNil)
1081         }
1082         mtxt, err := fs.MarshalManifest(".")
1083         c.Assert(err, check.IsNil)
1084         var coll arvados.Collection
1085         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1086                 "collection": map[string]string{
1087                         "manifest_text": mtxt,
1088                 },
1089         })
1090         c.Assert(err, check.IsNil)
1091
1092         for _, trial := range trials {
1093                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1094                 req := &http.Request{
1095                         Method:     "GET",
1096                         Host:       u.Host,
1097                         URL:        u,
1098                         RequestURI: u.RequestURI(),
1099                         Header: http.Header{
1100                                 "Authorization": {"Bearer " + client.AuthToken},
1101                         },
1102                 }
1103                 resp := httptest.NewRecorder()
1104                 s.testServer.Handler.ServeHTTP(resp, req)
1105                 c.Check(resp.Code, check.Equals, http.StatusOK)
1106                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1107                 c.Check(resp.Body.String(), check.Equals, trial.content)
1108         }
1109 }
1110
1111 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1112         s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1113         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1114         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1115         req := &http.Request{
1116                 Method:     "GET",
1117                 Host:       u.Host,
1118                 URL:        u,
1119                 RequestURI: u.RequestURI(),
1120         }
1121         resp := httptest.NewRecorder()
1122         s.testServer.Handler.ServeHTTP(resp, req)
1123         c.Check(resp.Code, check.Equals, http.StatusOK)
1124         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1125 }
1126
1127 // Writing to a collection shouldn't affect its entry in the
1128 // PDH-to-manifest cache.
1129 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1130         arv, err := arvadosclient.MakeArvadosClient()
1131         c.Assert(err, check.Equals, nil)
1132         arv.ApiToken = arvadostest.ActiveToken
1133
1134         u := mustParseURL("http://x.example/testfile")
1135         req := &http.Request{
1136                 Method:     "GET",
1137                 Host:       u.Host,
1138                 URL:        u,
1139                 RequestURI: u.RequestURI(),
1140                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1141         }
1142
1143         checkWithID := func(id string, status int) {
1144                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1145                 req.Host = req.URL.Host
1146                 resp := httptest.NewRecorder()
1147                 s.testServer.Handler.ServeHTTP(resp, req)
1148                 c.Check(resp.Code, check.Equals, status)
1149         }
1150
1151         var colls [2]arvados.Collection
1152         for i := range colls {
1153                 err := arv.Create("collections",
1154                         map[string]interface{}{
1155                                 "ensure_unique_name": true,
1156                                 "collection": map[string]interface{}{
1157                                         "name": "test collection",
1158                                 },
1159                         }, &colls[i])
1160                 c.Assert(err, check.Equals, nil)
1161         }
1162
1163         // Populate cache with empty collection
1164         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1165
1166         // write a file to colls[0]
1167         reqPut := *req
1168         reqPut.Method = "PUT"
1169         reqPut.URL.Host = colls[0].UUID + ".example"
1170         reqPut.Host = req.URL.Host
1171         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1172         resp := httptest.NewRecorder()
1173         s.testServer.Handler.ServeHTTP(resp, &reqPut)
1174         c.Check(resp.Code, check.Equals, http.StatusCreated)
1175
1176         // new file should not appear in colls[1]
1177         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1178         checkWithID(colls[1].UUID, http.StatusNotFound)
1179
1180         checkWithID(colls[0].UUID, http.StatusOK)
1181 }
1182
1183 func copyHeader(h http.Header) http.Header {
1184         hc := http.Header{}
1185         for k, v := range h {
1186                 hc[k] = append([]string(nil), v...)
1187         }
1188         return hc
1189 }
1190
1191 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, h *handler, req *http.Request,
1192         successCode int, direction string, perm bool, userUuid string, collectionUuid string, filepath string) {
1193
1194         client := s.testServer.Config.Client
1195         client.AuthToken = arvadostest.AdminToken
1196         var logentries arvados.LogList
1197         limit1 := 1
1198         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1199                 arvados.ResourceListParams{
1200                         Limit: &limit1,
1201                         Order: "created_at desc"})
1202         c.Check(err, check.IsNil)
1203         c.Check(logentries.Items, check.HasLen, 1)
1204         lastLogId := logentries.Items[0].ID
1205
1206         var logbuf bytes.Buffer
1207         logger := logrus.New()
1208         logger.Out = &logbuf
1209         resp := httptest.NewRecorder()
1210         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1211         h.ServeHTTP(resp, req)
1212
1213         if perm {
1214                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
1215                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
1216                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1217
1218                 deadline := time.Now().Add(time.Second)
1219                 for {
1220                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
1221                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
1222                                 arvados.ResourceListParams{
1223                                         Filters: []arvados.Filter{
1224                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
1225                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
1226                                         },
1227                                         Limit: &limit1,
1228                                         Order: "created_at desc",
1229                                 })
1230                         c.Assert(err, check.IsNil)
1231                         if len(logentries.Items) > 0 &&
1232                                 logentries.Items[0].ID > lastLogId &&
1233                                 logentries.Items[0].ObjectUUID == userUuid &&
1234                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
1235                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
1236                                 break
1237                         }
1238                         c.Logf("logentries.Items: %+v", logentries.Items)
1239                         time.Sleep(50 * time.Millisecond)
1240                 }
1241         } else {
1242                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1243                 c.Check(logbuf.String(), check.Equals, "")
1244         }
1245 }
1246
1247 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
1248         config := newConfig(ctxlog.TestLogger(c), s.ArvConfig)
1249         h := handler{Config: config}
1250         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1251
1252         config.cluster.Collections.TrustAllContent = true
1253
1254         for _, adminperm := range []bool{true, false} {
1255                 for _, userperm := range []bool{true, false} {
1256                         config.cluster.Collections.WebDAVPermission.Admin.Download = adminperm
1257                         config.cluster.Collections.WebDAVPermission.User.Download = userperm
1258
1259                         // Test admin permission
1260                         req := &http.Request{
1261                                 Method:     "GET",
1262                                 Host:       u.Host,
1263                                 URL:        u,
1264                                 RequestURI: u.RequestURI(),
1265                                 Header: http.Header{
1266                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1267                                 },
1268                         }
1269                         s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", adminperm,
1270                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, "foo")
1271
1272                         // Test user permission
1273                         req = &http.Request{
1274                                 Method:     "GET",
1275                                 Host:       u.Host,
1276                                 URL:        u,
1277                                 RequestURI: u.RequestURI(),
1278                                 Header: http.Header{
1279                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1280                                 },
1281                         }
1282                         s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", userperm,
1283                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
1284                 }
1285         }
1286
1287         config.cluster.Collections.WebDAVPermission.User.Download = true
1288
1289         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
1290                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
1291
1292                 u = mustParseURL(tryurl)
1293                 req := &http.Request{
1294                         Method:     "GET",
1295                         Host:       u.Host,
1296                         URL:        u,
1297                         RequestURI: u.RequestURI(),
1298                         Header: http.Header{
1299                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1300                         },
1301                 }
1302                 s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true,
1303                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, "dir1/subdir/file1")
1304         }
1305
1306         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
1307         req := &http.Request{
1308                 Method:     "GET",
1309                 Host:       u.Host,
1310                 URL:        u,
1311                 RequestURI: u.RequestURI(),
1312                 Header: http.Header{
1313                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1314                 },
1315         }
1316         s.checkUploadDownloadRequest(c, &h, req, http.StatusOK, "download", true,
1317                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, "foo")
1318 }
1319
1320 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
1321         config := newConfig(ctxlog.TestLogger(c), s.ArvConfig)
1322         h := handler{Config: config}
1323
1324         for _, adminperm := range []bool{true, false} {
1325                 for _, userperm := range []bool{true, false} {
1326
1327                         arv := s.testServer.Config.Client
1328                         arv.AuthToken = arvadostest.ActiveToken
1329
1330                         var coll arvados.Collection
1331                         err := arv.RequestAndDecode(&coll,
1332                                 "POST",
1333                                 "/arvados/v1/collections",
1334                                 nil,
1335                                 map[string]interface{}{
1336                                         "ensure_unique_name": true,
1337                                         "collection": map[string]interface{}{
1338                                                 "name": "test collection",
1339                                         },
1340                                 })
1341                         c.Assert(err, check.Equals, nil)
1342
1343                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
1344
1345                         config.cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
1346                         config.cluster.Collections.WebDAVPermission.User.Upload = userperm
1347
1348                         // Test admin permission
1349                         req := &http.Request{
1350                                 Method:     "PUT",
1351                                 Host:       u.Host,
1352                                 URL:        u,
1353                                 RequestURI: u.RequestURI(),
1354                                 Header: http.Header{
1355                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1356                                 },
1357                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1358                         }
1359                         s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", adminperm,
1360                                 arvadostest.AdminUserUUID, coll.UUID, "bar")
1361
1362                         // Test user permission
1363                         req = &http.Request{
1364                                 Method:     "PUT",
1365                                 Host:       u.Host,
1366                                 URL:        u,
1367                                 RequestURI: u.RequestURI(),
1368                                 Header: http.Header{
1369                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1370                                 },
1371                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1372                         }
1373                         s.checkUploadDownloadRequest(c, &h, req, http.StatusCreated, "upload", userperm,
1374                                 arvadostest.ActiveUserUUID, coll.UUID, "bar")
1375                 }
1376         }
1377 }