services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "time"
  21
  22         "git.arvados.org/arvados.git/lib/config"
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  25         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  26         "git.arvados.org/arvados.git/sdk/go/auth"
  27         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  28         "git.arvados.org/arvados.git/sdk/go/keepclient"
  29         "github.com/sirupsen/logrus"
  30         check "gopkg.in/check.v1"
  31 )
  32
  33 var _ = check.Suite(&UnitSuite{})
  34
  35 func init() {
  36         arvados.DebugLocksPanicMode = true
  37 }
  38
  39 type UnitSuite struct {
  40         Config *arvados.Config
  41 }
  42
  43 func (s *UnitSuite) SetUpTest(c *check.C) {
  44         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  45         ldr.Path = "-"
  46         cfg, err := ldr.Load()
  47         c.Assert(err, check.IsNil)
  48         s.Config = cfg
  49 }
  50
  51 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  52         h := handler{Config: newConfig(s.Config)}
  53         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  54         req := &http.Request{
  55                 Method:     "OPTIONS",
  56                 Host:       u.Host,
  57                 URL:        u,
  58                 RequestURI: u.RequestURI(),
  59                 Header: http.Header{
  60                         "Origin":                        {"https://workbench.example"},
  61                         "Access-Control-Request-Method": {"POST"},
  62                 },
  63         }
  64
  65         // Check preflight for an allowed request
  66         resp := httptest.NewRecorder()
  67         h.ServeHTTP(resp, req)
  68         c.Check(resp.Code, check.Equals, http.StatusOK)
  69         c.Check(resp.Body.String(), check.Equals, "")
  70         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  71         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  72         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  73
  74         // Check preflight for a disallowed request
  75         resp = httptest.NewRecorder()
  76         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  77         h.ServeHTTP(resp, req)
  78         c.Check(resp.Body.String(), check.Equals, "")
  79         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  80 }
  81
  82 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
  83         for _, trial := range []struct {
  84                 dataExists    bool
  85                 sendIMSHeader bool
  86                 expectStatus  int
  87                 logRegexp     string
  88         }{
  89                 // If we return no content due to a Keep read error,
  90                 // we should emit a log message.
  91                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
  92
  93                 // If we return no content because the client sent an
  94                 // If-Modified-Since header, our response should be
  95                 // 304, and we should not emit a log message.
  96                 {true, true, http.StatusNotModified, ``},
  97         } {
  98                 c.Logf("trial: %+v", trial)
  99                 arvadostest.StartKeep(2, true)
 100                 if trial.dataExists {
 101                         arv, err := arvadosclient.MakeArvadosClient()
 102                         c.Assert(err, check.IsNil)
 103                         arv.ApiToken = arvadostest.ActiveToken
 104                         kc, err := keepclient.MakeKeepClient(arv)
 105                         c.Assert(err, check.IsNil)
 106                         _, _, err = kc.PutB([]byte("foo"))
 107                         c.Assert(err, check.IsNil)
 108                 }
 109
 110                 h := handler{Config: newConfig(s.Config)}
 111                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 112                 req := &http.Request{
 113                         Method:     "GET",
 114                         Host:       u.Host,
 115                         URL:        u,
 116                         RequestURI: u.RequestURI(),
 117                         Header: http.Header{
 118                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 119                         },
 120                 }
 121                 if trial.sendIMSHeader {
 122                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 123                 }
 124
 125                 var logbuf bytes.Buffer
 126                 logger := logrus.New()
 127                 logger.Out = &logbuf
 128                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 129
 130                 resp := httptest.NewRecorder()
 131                 h.ServeHTTP(resp, req)
 132                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 133                 c.Check(resp.Body.String(), check.Equals, "")
 134
 135                 c.Log(logbuf.String())
 136                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 137         }
 138 }
 139
 140 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 141         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 142         token := arvadostest.ActiveToken
 143         for _, trial := range []string{
 144                 "http://keep-web/c=" + bogusID + "/foo",
 145                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 146                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 147                 "http://keep-web/collections/" + bogusID + "/foo",
 148                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 149                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 150         } {
 151                 c.Log(trial)
 152                 u := mustParseURL(trial)
 153                 req := &http.Request{
 154                         Method:     "GET",
 155                         Host:       u.Host,
 156                         URL:        u,
 157                         RequestURI: u.RequestURI(),
 158                 }
 159                 resp := httptest.NewRecorder()
 160                 cfg := newConfig(s.Config)
 161                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 162                 h := handler{Config: cfg}
 163                 h.ServeHTTP(resp, req)
 164                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 165         }
 166 }
 167
 168 func mustParseURL(s string) *url.URL {
 169         r, err := url.Parse(s)
 170         if err != nil {
 171                 panic("parse URL: " + s)
 172         }
 173         return r
 174 }
 175
 176 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 177         for _, testURL := range []string{
 178                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 179                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 180         } {
 181                 resp := httptest.NewRecorder()
 182                 u := mustParseURL(testURL)
 183                 req := &http.Request{
 184                         Method:     "GET",
 185                         URL:        u,
 186                         RequestURI: u.RequestURI(),
 187                 }
 188                 s.testServer.Handler.ServeHTTP(resp, req)
 189                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 190                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 191         }
 192 }
 193
 194 // An authorizer modifies an HTTP request to make use of the given
 195 // token -- by adding it to a header, cookie, query param, or whatever
 196 // -- and returns the HTTP status code we should expect from keep-web if
 197 // the token is invalid.
 198 type authorizer func(*http.Request, string) int
 199
 200 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 201         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 202 }
 203 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 204         r.Header.Add("Authorization", "Bearer "+tok)
 205         return http.StatusUnauthorized
 206 }
 207 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 208         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 209 }
 210 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 211         r.Header.Add("Authorization", "Bearer "+tok)
 212         return http.StatusUnauthorized
 213 }
 214
 215 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 216         s.doVhostRequests(c, authzViaCookieValue)
 217 }
 218 func authzViaCookieValue(r *http.Request, tok string) int {
 219         r.AddCookie(&http.Cookie{
 220                 Name:  "arvados_api_token",
 221                 Value: auth.EncodeTokenCookie([]byte(tok)),
 222         })
 223         return http.StatusUnauthorized
 224 }
 225
 226 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 227         s.doVhostRequests(c, authzViaPath)
 228 }
 229 func authzViaPath(r *http.Request, tok string) int {
 230         r.URL.Path = "/t=" + tok + r.URL.Path
 231         return http.StatusNotFound
 232 }
 233
 234 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 235         s.doVhostRequests(c, authzViaQueryString)
 236 }
 237 func authzViaQueryString(r *http.Request, tok string) int {
 238         r.URL.RawQuery = "api_token=" + tok
 239         return http.StatusUnauthorized
 240 }
 241
 242 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 243         s.doVhostRequests(c, authzViaPOST)
 244 }
 245 func authzViaPOST(r *http.Request, tok string) int {
 246         r.Method = "POST"
 247         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 248         r.Body = ioutil.NopCloser(strings.NewReader(
 249                 url.Values{"api_token": {tok}}.Encode()))
 250         return http.StatusUnauthorized
 251 }
 252
 253 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 254         s.doVhostRequests(c, authzViaPOST)
 255 }
 256 func authzViaXHRPOST(r *http.Request, tok string) int {
 257         r.Method = "POST"
 258         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 259         r.Header.Add("Origin", "https://origin.example")
 260         r.Body = ioutil.NopCloser(strings.NewReader(
 261                 url.Values{
 262                         "api_token":   {tok},
 263                         "disposition": {"attachment"},
 264                 }.Encode()))
 265         return http.StatusUnauthorized
 266 }
 267
 268 // Try some combinations of {url, token} using the given authorization
 269 // mechanism, and verify the result is correct.
 270 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 271         for _, hostPath := range []string{
 272                 arvadostest.FooCollection + ".example.com/foo",
 273                 arvadostest.FooCollection + "--collections.example.com/foo",
 274                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 275                 arvadostest.FooCollectionPDH + ".example.com/foo",
 276                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 277                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 278         } {
 279                 c.Log("doRequests: ", hostPath)
 280                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 281         }
 282 }
 283
 284 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 285         for _, tok := range []string{
 286                 arvadostest.ActiveToken,
 287                 arvadostest.ActiveToken[:15],
 288                 arvadostest.SpectatorToken,
 289                 "bogus",
 290                 "",
 291         } {
 292                 u := mustParseURL("http://" + hostPath)
 293                 req := &http.Request{
 294                         Method:     "GET",
 295                         Host:       u.Host,
 296                         URL:        u,
 297                         RequestURI: u.RequestURI(),
 298                         Header:     http.Header{},
 299                 }
 300                 failCode := authz(req, tok)
 301                 req, resp := s.doReq(req)
 302                 code, body := resp.Code, resp.Body.String()
 303
 304                 // If the initial request had a (non-empty) token
 305                 // showing in the query string, we should have been
 306                 // redirected in order to hide it in a cookie.
 307                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 308
 309                 if tok == arvadostest.ActiveToken {
 310                         c.Check(code, check.Equals, http.StatusOK)
 311                         c.Check(body, check.Equals, "foo")
 312                 } else {
 313                         c.Check(code >= 400, check.Equals, true)
 314                         c.Check(code < 500, check.Equals, true)
 315                         if tok == arvadostest.SpectatorToken {
 316                                 // Valid token never offers to retry
 317                                 // with different credentials.
 318                                 c.Check(code, check.Equals, http.StatusNotFound)
 319                         } else {
 320                                 // Invalid token can ask to retry
 321                                 // depending on the authz method.
 322                                 c.Check(code, check.Equals, failCode)
 323                         }
 324                         if code == 404 {
 325                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 326                         } else {
 327                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 328                         }
 329                 }
 330         }
 331 }
 332
 333 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 334         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 335                 for _, port := range []string{"80", "443", "8000"} {
 336                         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 337                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 338                         req := &http.Request{
 339                                 Method:     "GET",
 340                                 Host:       u.Host,
 341                                 URL:        u,
 342                                 RequestURI: u.RequestURI(),
 343                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 344                         }
 345                         req, resp := s.doReq(req)
 346                         code, _ := resp.Code, resp.Body.String()
 347
 348                         if port == "8000" {
 349                                 c.Check(code, check.Equals, 401)
 350                         } else {
 351                                 c.Check(code, check.Equals, 200)
 352                         }
 353                 }
 354         }
 355 }
 356
 357 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 358         resp := httptest.NewRecorder()
 359         s.testServer.Handler.ServeHTTP(resp, req)
 360         if resp.Code != http.StatusSeeOther {
 361                 return req, resp
 362         }
 363         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 364         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 365         req = &http.Request{
 366                 Method:     "GET",
 367                 Host:       u.Host,
 368                 URL:        u,
 369                 RequestURI: u.RequestURI(),
 370                 Header:     http.Header{},
 371         }
 372         for _, c := range cookies {
 373                 req.AddCookie(c)
 374         }
 375         return s.doReq(req)
 376 }
 377
 378 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 379         s.testVhostRedirectTokenToCookie(c, "GET",
 380                 arvadostest.FooCollection+".example.com/foo",
 381                 "?api_token="+arvadostest.ActiveToken,
 382                 "",
 383                 "",
 384                 http.StatusOK,
 385                 "foo",
 386         )
 387 }
 388
 389 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 390         s.testVhostRedirectTokenToCookie(c, "GET",
 391                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 392                 "",
 393                 "",
 394                 "",
 395                 http.StatusOK,
 396                 "foo",
 397         )
 398 }
 399
 400 // Bad token in URL is 404 Not Found because it doesn't make sense to
 401 // retry the same URL with different authorization.
 402 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 403         s.testVhostRedirectTokenToCookie(c, "GET",
 404                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 405                 "",
 406                 "",
 407                 "",
 408                 http.StatusNotFound,
 409                 notFoundMessage+"\n",
 410         )
 411 }
 412
 413 // Bad token in a cookie (even if it got there via our own
 414 // query-string-to-cookie redirect) is, in principle, retryable at the
 415 // same URL so it's 401 Unauthorized.
 416 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 417         s.testVhostRedirectTokenToCookie(c, "GET",
 418                 arvadostest.FooCollection+".example.com/foo",
 419                 "?api_token=thisisabogustoken",
 420                 "",
 421                 "",
 422                 http.StatusUnauthorized,
 423                 unauthorizedMessage+"\n",
 424         )
 425 }
 426
 427 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 428         s.testVhostRedirectTokenToCookie(c, "GET",
 429                 "example.com/c="+arvadostest.FooCollection+"/foo",
 430                 "?api_token="+arvadostest.ActiveToken,
 431                 "",
 432                 "",
 433                 http.StatusBadRequest,
 434                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 435         )
 436 }
 437
 438 // If client requests an attachment by putting ?disposition=attachment
 439 // in the query string, and gets redirected, the redirect target
 440 // should respond with an attachment.
 441 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 442         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 443                 arvadostest.FooCollection+".example.com/foo",
 444                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 445                 "",
 446                 "",
 447                 http.StatusOK,
 448                 "foo",
 449         )
 450         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 451 }
 452
 453 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 454         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 455         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 456                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 457                 "?api_token="+arvadostest.ActiveToken,
 458                 "",
 459                 "",
 460                 http.StatusOK,
 461                 "foo",
 462         )
 463         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 464 }
 465
 466 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 467         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 468         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 469                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 470                 "?api_token="+arvadostest.ActiveToken,
 471                 "",
 472                 "",
 473                 http.StatusOK,
 474                 "waz",
 475         )
 476         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 477         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 478                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 479                 "?api_token="+arvadostest.ActiveToken,
 480                 "",
 481                 "",
 482                 http.StatusOK,
 483                 "waz",
 484         )
 485         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 486 }
 487
 488 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 489         s.testServer.Config.cluster.Collections.TrustAllContent = true
 490         s.testVhostRedirectTokenToCookie(c, "GET",
 491                 "example.com/c="+arvadostest.FooCollection+"/foo",
 492                 "?api_token="+arvadostest.ActiveToken,
 493                 "",
 494                 "",
 495                 http.StatusOK,
 496                 "foo",
 497         )
 498 }
 499
 500 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 501         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 502
 503         s.testVhostRedirectTokenToCookie(c, "GET",
 504                 "example.com/c="+arvadostest.FooCollection+"/foo",
 505                 "?api_token="+arvadostest.ActiveToken,
 506                 "",
 507                 "",
 508                 http.StatusBadRequest,
 509                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 510         )
 511
 512         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 513                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 514                 "?api_token="+arvadostest.ActiveToken,
 515                 "",
 516                 "",
 517                 http.StatusOK,
 518                 "foo",
 519         )
 520         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 521 }
 522
 523 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 524         s.testVhostRedirectTokenToCookie(c, "POST",
 525                 arvadostest.FooCollection+".example.com/foo",
 526                 "",
 527                 "application/x-www-form-urlencoded",
 528                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 529                 http.StatusOK,
 530                 "foo",
 531         )
 532 }
 533
 534 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 535         s.testVhostRedirectTokenToCookie(c, "POST",
 536                 arvadostest.FooCollection+".example.com/foo",
 537                 "",
 538                 "application/x-www-form-urlencoded",
 539                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 540                 http.StatusNotFound,
 541                 notFoundMessage+"\n",
 542         )
 543 }
 544
 545 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 546         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 547         s.testVhostRedirectTokenToCookie(c, "GET",
 548                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 549                 "",
 550                 "",
 551                 "",
 552                 http.StatusOK,
 553                 "Hello world\n",
 554         )
 555 }
 556
 557 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 558         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 559         s.testVhostRedirectTokenToCookie(c, "GET",
 560                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 561                 "",
 562                 "",
 563                 "",
 564                 http.StatusNotFound,
 565                 notFoundMessage+"\n",
 566         )
 567 }
 568
 569 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 570         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 571
 572         client := s.testServer.Config.Client
 573         client.AuthToken = arvadostest.ActiveToken
 574         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 575         c.Assert(err, check.IsNil)
 576         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 577         c.Assert(err, check.IsNil)
 578         f.Close()
 579         mtxt, err := fs.MarshalManifest(".")
 580         c.Assert(err, check.IsNil)
 581         var coll arvados.Collection
 582         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 583                 "collection": map[string]string{
 584                         "manifest_text": mtxt,
 585                 },
 586         })
 587         c.Assert(err, check.IsNil)
 588
 589         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 590         req := &http.Request{
 591                 Method:     "GET",
 592                 Host:       u.Host,
 593                 URL:        u,
 594                 RequestURI: u.RequestURI(),
 595                 Header: http.Header{
 596                         "Authorization": {"Bearer " + client.AuthToken},
 597                 },
 598         }
 599         resp := httptest.NewRecorder()
 600         s.testServer.Handler.ServeHTTP(resp, req)
 601         c.Check(resp.Code, check.Equals, http.StatusOK)
 602         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 603 }
 604
 605 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 606         arv := arvados.NewClientFromEnv()
 607         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 608         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 609         name := "foo/bar/baz"
 610         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 611         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 612
 613         client := s.testServer.Config.Client
 614         client.AuthToken = arvadostest.ActiveToken
 615         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 616         c.Assert(err, check.IsNil)
 617         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 618         c.Assert(err, check.IsNil)
 619         f.Close()
 620         mtxt, err := fs.MarshalManifest(".")
 621         c.Assert(err, check.IsNil)
 622         var coll arvados.Collection
 623         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 624                 "collection": map[string]string{
 625                         "manifest_text": mtxt,
 626                         "name":          name,
 627                         "owner_uuid":    arvadostest.AProjectUUID,
 628                 },
 629         })
 630         c.Assert(err, check.IsNil)
 631         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 632
 633         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 634         for tryURL, expectRegexp := range map[string]string{
 635                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 636                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 637         } {
 638                 u, _ := url.Parse(tryURL)
 639                 req := &http.Request{
 640                         Method:     "GET",
 641                         Host:       u.Host,
 642                         URL:        u,
 643                         RequestURI: u.RequestURI(),
 644                         Header: http.Header{
 645                                 "Authorization": {"Bearer " + client.AuthToken},
 646                         },
 647                 }
 648                 resp := httptest.NewRecorder()
 649                 s.testServer.Handler.ServeHTTP(resp, req)
 650                 c.Check(resp.Code, check.Equals, http.StatusOK)
 651                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 652         }
 653 }
 654
 655 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 656 // and disposition=attachment (telling us it's acceptable to respond
 657 // with content instead of a redirect) and an Origin header that gets
 658 // added automatically by the browser (telling us it's desirable to do
 659 // so).
 660 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 661         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 662         req := &http.Request{
 663                 Method:     "POST",
 664                 Host:       u.Host,
 665                 URL:        u,
 666                 RequestURI: u.RequestURI(),
 667                 Header: http.Header{
 668                         "Origin":       {"https://origin.example"},
 669                         "Content-Type": {"application/x-www-form-urlencoded"},
 670                 },
 671                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 672                         "api_token":   {arvadostest.ActiveToken},
 673                         "disposition": {"attachment"},
 674                 }.Encode())),
 675         }
 676         resp := httptest.NewRecorder()
 677         s.testServer.Handler.ServeHTTP(resp, req)
 678         c.Check(resp.Code, check.Equals, http.StatusOK)
 679         c.Check(resp.Body.String(), check.Equals, "foo")
 680         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 681
 682         // GET + Origin header is representative of both AJAX GET
 683         // requests and inline images via <IMG crossorigin="anonymous"
 684         // src="...">.
 685         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 686         req = &http.Request{
 687                 Method:     "GET",
 688                 Host:       u.Host,
 689                 URL:        u,
 690                 RequestURI: u.RequestURI(),
 691                 Header: http.Header{
 692                         "Origin": {"https://origin.example"},
 693                 },
 694         }
 695         resp = httptest.NewRecorder()
 696         s.testServer.Handler.ServeHTTP(resp, req)
 697         c.Check(resp.Code, check.Equals, http.StatusOK)
 698         c.Check(resp.Body.String(), check.Equals, "foo")
 699         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 700 }
 701
 702 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 703         u, _ := url.Parse(`http://` + hostPath + queryString)
 704         req := &http.Request{
 705                 Method:     method,
 706                 Host:       u.Host,
 707                 URL:        u,
 708                 RequestURI: u.RequestURI(),
 709                 Header:     http.Header{"Content-Type": {contentType}},
 710                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 711         }
 712
 713         resp := httptest.NewRecorder()
 714         defer func() {
 715                 c.Check(resp.Code, check.Equals, expectStatus)
 716                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 717         }()
 718
 719         s.testServer.Handler.ServeHTTP(resp, req)
 720         if resp.Code != http.StatusSeeOther {
 721                 return resp
 722         }
 723         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 724         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 725
 726         u, _ = u.Parse(resp.Header().Get("Location"))
 727         req = &http.Request{
 728                 Method:     "GET",
 729                 Host:       u.Host,
 730                 URL:        u,
 731                 RequestURI: u.RequestURI(),
 732                 Header:     http.Header{},
 733         }
 734         for _, c := range cookies {
 735                 req.AddCookie(c)
 736         }
 737
 738         resp = httptest.NewRecorder()
 739         s.testServer.Handler.ServeHTTP(resp, req)
 740         c.Check(resp.Header().Get("Location"), check.Equals, "")
 741         return resp
 742 }
 743
 744 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 745         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 746         s.testDirectoryListing(c)
 747 }
 748
 749 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 750         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 751         s.testDirectoryListing(c)
 752 }
 753
 754 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 755         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 756         authHeader := http.Header{
 757                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 758         }
 759         for _, trial := range []struct {
 760                 uri      string
 761                 header   http.Header
 762                 expect   []string
 763                 redirect string
 764                 cutDirs  int
 765         }{
 766                 {
 767                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 768                         header:  authHeader,
 769                         expect:  []string{"dir1/foo", "dir1/bar"},
 770                         cutDirs: 0,
 771                 },
 772                 {
 773                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 774                         header:  authHeader,
 775                         expect:  []string{"foo", "bar"},
 776                         cutDirs: 1,
 777                 },
 778                 {
 779                         // URLs of this form ignore authHeader, and
 780                         // FooAndBarFilesInDirUUID isn't public, so
 781                         // this returns 401.
 782                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 783                         header: authHeader,
 784                         expect: nil,
 785                 },
 786                 {
 787                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 788                         header:  authHeader,
 789                         expect:  []string{"dir1/"},
 790                         cutDirs: 3,
 791                 },
 792                 {
 793                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 794                         header:  authHeader,
 795                         expect:  []string{"bar"},
 796                         cutDirs: 4,
 797                 },
 798                 {
 799                         uri:     "download.example.com/",
 800                         header:  authHeader,
 801                         expect:  []string{"users/"},
 802                         cutDirs: 0,
 803                 },
 804                 {
 805                         uri:      "download.example.com/users",
 806                         header:   authHeader,
 807                         redirect: "/users/",
 808                         expect:   []string{"active/"},
 809                         cutDirs:  1,
 810                 },
 811                 {
 812                         uri:     "download.example.com/users/",
 813                         header:  authHeader,
 814                         expect:  []string{"active/"},
 815                         cutDirs: 1,
 816                 },
 817                 {
 818                         uri:      "download.example.com/users/active",
 819                         header:   authHeader,
 820                         redirect: "/users/active/",
 821                         expect:   []string{"foo_file_in_dir/"},
 822                         cutDirs:  2,
 823                 },
 824                 {
 825                         uri:     "download.example.com/users/active/",
 826                         header:  authHeader,
 827                         expect:  []string{"foo_file_in_dir/"},
 828                         cutDirs: 2,
 829                 },
 830                 {
 831                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 832                         header:  nil,
 833                         expect:  []string{"dir1/foo", "dir1/bar"},
 834                         cutDirs: 4,
 835                 },
 836                 {
 837                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 838                         header:  nil,
 839                         expect:  []string{"dir1/foo", "dir1/bar"},
 840                         cutDirs: 2,
 841                 },
 842                 {
 843                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 844                         header:  nil,
 845                         expect:  []string{"dir1/foo", "dir1/bar"},
 846                         cutDirs: 2,
 847                 },
 848                 {
 849                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 850                         header:  authHeader,
 851                         expect:  []string{"dir1/foo", "dir1/bar"},
 852                         cutDirs: 1,
 853                 },
 854                 {
 855                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 856                         header:   authHeader,
 857                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 858                         expect:   []string{"foo", "bar"},
 859                         cutDirs:  2,
 860                 },
 861                 {
 862                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 863                         header:  authHeader,
 864                         expect:  []string{"foo", "bar"},
 865                         cutDirs: 3,
 866                 },
 867                 {
 868                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 869                         header:   authHeader,
 870                         redirect: "/dir1/",
 871                         expect:   []string{"foo", "bar"},
 872                         cutDirs:  1,
 873                 },
 874                 {
 875                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 876                         header: authHeader,
 877                         expect: nil,
 878                 },
 879                 {
 880                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 881                         header:  authHeader,
 882                         expect:  []string{"waz"},
 883                         cutDirs: 1,
 884                 },
 885                 {
 886                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 887                         header:  authHeader,
 888                         expect:  []string{"waz"},
 889                         cutDirs: 2,
 890                 },
 891         } {
 892                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 893                 resp := httptest.NewRecorder()
 894                 u := mustParseURL("//" + trial.uri)
 895                 req := &http.Request{
 896                         Method:     "GET",
 897                         Host:       u.Host,
 898                         URL:        u,
 899                         RequestURI: u.RequestURI(),
 900                         Header:     copyHeader(trial.header),
 901                 }
 902                 s.testServer.Handler.ServeHTTP(resp, req)
 903                 var cookies []*http.Cookie
 904                 for resp.Code == http.StatusSeeOther {
 905                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 906                         req = &http.Request{
 907                                 Method:     "GET",
 908                                 Host:       u.Host,
 909                                 URL:        u,
 910                                 RequestURI: u.RequestURI(),
 911                                 Header:     copyHeader(trial.header),
 912                         }
 913                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 914                         for _, c := range cookies {
 915                                 req.AddCookie(c)
 916                         }
 917                         resp = httptest.NewRecorder()
 918                         s.testServer.Handler.ServeHTTP(resp, req)
 919                 }
 920                 if trial.redirect != "" {
 921                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 922                 }
 923                 if trial.expect == nil {
 924                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 925                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 926                         } else {
 927                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 928                         }
 929                 } else {
 930                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 931                         for _, e := range trial.expect {
 932                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 933                         }
 934                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 935                 }
 936
 937                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 938                 req = &http.Request{
 939                         Method:     "OPTIONS",
 940                         Host:       u.Host,
 941                         URL:        u,
 942                         RequestURI: u.RequestURI(),
 943                         Header:     copyHeader(trial.header),
 944                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 945                 }
 946                 resp = httptest.NewRecorder()
 947                 s.testServer.Handler.ServeHTTP(resp, req)
 948                 if trial.expect == nil {
 949                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 950                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 951                         } else {
 952                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 953                         }
 954                 } else {
 955                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 956                 }
 957
 958                 req = &http.Request{
 959                         Method:     "PROPFIND",
 960                         Host:       u.Host,
 961                         URL:        u,
 962                         RequestURI: u.RequestURI(),
 963                         Header:     copyHeader(trial.header),
 964                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 965                 }
 966                 resp = httptest.NewRecorder()
 967                 s.testServer.Handler.ServeHTTP(resp, req)
 968                 if trial.expect == nil {
 969                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 970                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 971                         } else {
 972                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 973                         }
 974                 } else {
 975                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 976                         for _, e := range trial.expect {
 977                                 if strings.HasSuffix(e, "/") {
 978                                         e = filepath.Join(u.Path, e) + "/"
 979                                 } else {
 980                                         e = filepath.Join(u.Path, e)
 981                                 }
 982                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 983                         }
 984                 }
 985         }
 986 }
 987
 988 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 989         arv := arvados.NewClientFromEnv()
 990         var newCollection arvados.Collection
 991         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 992                 "collection": map[string]string{
 993                         "owner_uuid":    arvadostest.ActiveUserUUID,
 994                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 995                         "name":          "keep-web test collection",
 996                 },
 997                 "ensure_unique_name": true,
 998         })
 999         c.Assert(err, check.IsNil)
1000         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1001
1002         var updated arvados.Collection
1003         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1004                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1005                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1006                 req := &http.Request{
1007                         Method:     "DELETE",
1008                         Host:       u.Host,
1009                         URL:        u,
1010                         RequestURI: u.RequestURI(),
1011                         Header: http.Header{
1012                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1013                         },
1014                 }
1015                 resp := httptest.NewRecorder()
1016                 s.testServer.Handler.ServeHTTP(resp, req)
1017                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1018
1019                 updated = arvados.Collection{}
1020                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1021                 c.Check(err, check.IsNil)
1022                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1023                 c.Logf("updated manifest_text %q", updated.ManifestText)
1024         }
1025         c.Check(updated.ManifestText, check.Equals, "")
1026 }
1027
1028 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
1029         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
1030         authHeader := http.Header{
1031                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
1032         }
1033
1034         resp := httptest.NewRecorder()
1035         u := mustParseURL("http://download.example.com/_health/ping")
1036         req := &http.Request{
1037                 Method:     "GET",
1038                 Host:       u.Host,
1039                 URL:        u,
1040                 RequestURI: u.RequestURI(),
1041                 Header:     authHeader,
1042         }
1043         s.testServer.Handler.ServeHTTP(resp, req)
1044
1045         c.Check(resp.Code, check.Equals, http.StatusOK)
1046         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
1047 }
1048
1049 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1050         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1051
1052         client := s.testServer.Config.Client
1053         client.AuthToken = arvadostest.ActiveToken
1054         arv, err := arvadosclient.New(&client)
1055         c.Assert(err, check.Equals, nil)
1056         kc, err := keepclient.MakeKeepClient(arv)
1057         c.Assert(err, check.Equals, nil)
1058
1059         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
1060         c.Assert(err, check.IsNil)
1061
1062         trials := []struct {
1063                 filename    string
1064                 content     string
1065                 contentType string
1066         }{
1067                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1068                 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
1069                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1070                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1071                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1072         }
1073         for _, trial := range trials {
1074                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1075                 c.Assert(err, check.IsNil)
1076                 _, err = f.Write([]byte(trial.content))
1077                 c.Assert(err, check.IsNil)
1078                 c.Assert(f.Close(), check.IsNil)
1079         }
1080         mtxt, err := fs.MarshalManifest(".")
1081         c.Assert(err, check.IsNil)
1082         var coll arvados.Collection
1083         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1084                 "collection": map[string]string{
1085                         "manifest_text": mtxt,
1086                 },
1087         })
1088         c.Assert(err, check.IsNil)
1089
1090         for _, trial := range trials {
1091                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1092                 req := &http.Request{
1093                         Method:     "GET",
1094                         Host:       u.Host,
1095                         URL:        u,
1096                         RequestURI: u.RequestURI(),
1097                         Header: http.Header{
1098                                 "Authorization": {"Bearer " + client.AuthToken},
1099                         },
1100                 }
1101                 resp := httptest.NewRecorder()
1102                 s.testServer.Handler.ServeHTTP(resp, req)
1103                 c.Check(resp.Code, check.Equals, http.StatusOK)
1104                 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
1105                 c.Check(resp.Body.String(), check.Equals, trial.content)
1106         }
1107 }
1108
1109 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1110         s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1111         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1112         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1113         req := &http.Request{
1114                 Method:     "GET",
1115                 Host:       u.Host,
1116                 URL:        u,
1117                 RequestURI: u.RequestURI(),
1118         }
1119         resp := httptest.NewRecorder()
1120         s.testServer.Handler.ServeHTTP(resp, req)
1121         c.Check(resp.Code, check.Equals, http.StatusOK)
1122         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1123 }
1124
1125 // Writing to a collection shouldn't affect its entry in the
1126 // PDH-to-manifest cache.
1127 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1128         arv, err := arvadosclient.MakeArvadosClient()
1129         c.Assert(err, check.Equals, nil)
1130         arv.ApiToken = arvadostest.ActiveToken
1131
1132         u := mustParseURL("http://x.example/testfile")
1133         req := &http.Request{
1134                 Method:     "GET",
1135                 Host:       u.Host,
1136                 URL:        u,
1137                 RequestURI: u.RequestURI(),
1138                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1139         }
1140
1141         checkWithID := func(id string, status int) {
1142                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1143                 req.Host = req.URL.Host
1144                 resp := httptest.NewRecorder()
1145                 s.testServer.Handler.ServeHTTP(resp, req)
1146                 c.Check(resp.Code, check.Equals, status)
1147         }
1148
1149         var colls [2]arvados.Collection
1150         for i := range colls {
1151                 err := arv.Create("collections",
1152                         map[string]interface{}{
1153                                 "ensure_unique_name": true,
1154                                 "collection": map[string]interface{}{
1155                                         "name": "test collection",
1156                                 },
1157                         }, &colls[i])
1158                 c.Assert(err, check.Equals, nil)
1159         }
1160
1161         // Populate cache with empty collection
1162         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1163
1164         // write a file to colls[0]
1165         reqPut := *req
1166         reqPut.Method = "PUT"
1167         reqPut.URL.Host = colls[0].UUID + ".example"
1168         reqPut.Host = req.URL.Host
1169         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1170         resp := httptest.NewRecorder()
1171         s.testServer.Handler.ServeHTTP(resp, &reqPut)
1172         c.Check(resp.Code, check.Equals, http.StatusCreated)
1173
1174         // new file should not appear in colls[1]
1175         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1176         checkWithID(colls[1].UUID, http.StatusNotFound)
1177
1178         checkWithID(colls[0].UUID, http.StatusOK)
1179 }
1180
1181 func copyHeader(h http.Header) http.Header {
1182         hc := http.Header{}
1183         for k, v := range h {
1184                 hc[k] = append([]string(nil), v...)
1185         }
1186         return hc
1187 }