services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "time"
  21
  22         "git.arvados.org/arvados.git/lib/config"
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  25         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  26         "git.arvados.org/arvados.git/sdk/go/auth"
  27         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  28         "git.arvados.org/arvados.git/sdk/go/keepclient"
  29         "github.com/sirupsen/logrus"
  30         check "gopkg.in/check.v1"
  31 )
  32
  33 var _ = check.Suite(&UnitSuite{})
  34
  35 type UnitSuite struct {
  36         Config *arvados.Config
  37 }
  38
  39 func (s *UnitSuite) SetUpTest(c *check.C) {
  40         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  41         ldr.Path = "-"
  42         cfg, err := ldr.Load()
  43         c.Assert(err, check.IsNil)
  44         s.Config = cfg
  45 }
  46
  47 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  48         h := handler{Config: newConfig(s.Config)}
  49         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  50         req := &http.Request{
  51                 Method:     "OPTIONS",
  52                 Host:       u.Host,
  53                 URL:        u,
  54                 RequestURI: u.RequestURI(),
  55                 Header: http.Header{
  56                         "Origin":                        {"https://workbench.example"},
  57                         "Access-Control-Request-Method": {"POST"},
  58                 },
  59         }
  60
  61         // Check preflight for an allowed request
  62         resp := httptest.NewRecorder()
  63         h.ServeHTTP(resp, req)
  64         c.Check(resp.Code, check.Equals, http.StatusOK)
  65         c.Check(resp.Body.String(), check.Equals, "")
  66         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  67         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  68         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  69
  70         // Check preflight for a disallowed request
  71         resp = httptest.NewRecorder()
  72         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  73         h.ServeHTTP(resp, req)
  74         c.Check(resp.Body.String(), check.Equals, "")
  75         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  76 }
  77
  78 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
  79         for _, trial := range []struct {
  80                 dataExists    bool
  81                 sendIMSHeader bool
  82                 expectStatus  int
  83                 logRegexp     string
  84         }{
  85                 // If we return no content due to a Keep read error,
  86                 // we should emit a log message.
  87                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
  88
  89                 // If we return no content because the client sent an
  90                 // If-Modified-Since header, our response should be
  91                 // 304, and we should not emit a log message.
  92                 {true, true, http.StatusNotModified, ``},
  93         } {
  94                 c.Logf("trial: %+v", trial)
  95                 arvadostest.StartKeep(2, true)
  96                 if trial.dataExists {
  97                         arv, err := arvadosclient.MakeArvadosClient()
  98                         c.Assert(err, check.IsNil)
  99                         arv.ApiToken = arvadostest.ActiveToken
 100                         kc, err := keepclient.MakeKeepClient(arv)
 101                         c.Assert(err, check.IsNil)
 102                         _, _, err = kc.PutB([]byte("foo"))
 103                         c.Assert(err, check.IsNil)
 104                 }
 105
 106                 h := handler{Config: newConfig(s.Config)}
 107                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 108                 req := &http.Request{
 109                         Method:     "GET",
 110                         Host:       u.Host,
 111                         URL:        u,
 112                         RequestURI: u.RequestURI(),
 113                         Header: http.Header{
 114                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 115                         },
 116                 }
 117                 if trial.sendIMSHeader {
 118                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 119                 }
 120
 121                 var logbuf bytes.Buffer
 122                 logger := logrus.New()
 123                 logger.Out = &logbuf
 124                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 125
 126                 resp := httptest.NewRecorder()
 127                 h.ServeHTTP(resp, req)
 128                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 129                 c.Check(resp.Body.String(), check.Equals, "")
 130
 131                 c.Log(logbuf.String())
 132                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 133         }
 134 }
 135
 136 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 137         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 138         token := arvadostest.ActiveToken
 139         for _, trial := range []string{
 140                 "http://keep-web/c=" + bogusID + "/foo",
 141                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 142                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 143                 "http://keep-web/collections/" + bogusID + "/foo",
 144                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 145                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 146         } {
 147                 c.Log(trial)
 148                 u := mustParseURL(trial)
 149                 req := &http.Request{
 150                         Method:     "GET",
 151                         Host:       u.Host,
 152                         URL:        u,
 153                         RequestURI: u.RequestURI(),
 154                 }
 155                 resp := httptest.NewRecorder()
 156                 cfg := newConfig(s.Config)
 157                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 158                 h := handler{Config: cfg}
 159                 h.ServeHTTP(resp, req)
 160                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 161         }
 162 }
 163
 164 func mustParseURL(s string) *url.URL {
 165         r, err := url.Parse(s)
 166         if err != nil {
 167                 panic("parse URL: " + s)
 168         }
 169         return r
 170 }
 171
 172 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 173         for _, testURL := range []string{
 174                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 175                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 176         } {
 177                 resp := httptest.NewRecorder()
 178                 u := mustParseURL(testURL)
 179                 req := &http.Request{
 180                         Method:     "GET",
 181                         URL:        u,
 182                         RequestURI: u.RequestURI(),
 183                 }
 184                 s.testServer.Handler.ServeHTTP(resp, req)
 185                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 186                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 187         }
 188 }
 189
 190 // An authorizer modifies an HTTP request to make use of the given
 191 // token -- by adding it to a header, cookie, query param, or whatever
 192 // -- and returns the HTTP status code we should expect from keep-web if
 193 // the token is invalid.
 194 type authorizer func(*http.Request, string) int
 195
 196 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 197         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 198 }
 199 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 200         r.Header.Add("Authorization", "Bearer "+tok)
 201         return http.StatusUnauthorized
 202 }
 203 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 204         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 205 }
 206 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 207         r.Header.Add("Authorization", "Bearer "+tok)
 208         return http.StatusUnauthorized
 209 }
 210
 211 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 212         s.doVhostRequests(c, authzViaCookieValue)
 213 }
 214 func authzViaCookieValue(r *http.Request, tok string) int {
 215         r.AddCookie(&http.Cookie{
 216                 Name:  "arvados_api_token",
 217                 Value: auth.EncodeTokenCookie([]byte(tok)),
 218         })
 219         return http.StatusUnauthorized
 220 }
 221
 222 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 223         s.doVhostRequests(c, authzViaPath)
 224 }
 225 func authzViaPath(r *http.Request, tok string) int {
 226         r.URL.Path = "/t=" + tok + r.URL.Path
 227         return http.StatusNotFound
 228 }
 229
 230 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 231         s.doVhostRequests(c, authzViaQueryString)
 232 }
 233 func authzViaQueryString(r *http.Request, tok string) int {
 234         r.URL.RawQuery = "api_token=" + tok
 235         return http.StatusUnauthorized
 236 }
 237
 238 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 239         s.doVhostRequests(c, authzViaPOST)
 240 }
 241 func authzViaPOST(r *http.Request, tok string) int {
 242         r.Method = "POST"
 243         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 244         r.Body = ioutil.NopCloser(strings.NewReader(
 245                 url.Values{"api_token": {tok}}.Encode()))
 246         return http.StatusUnauthorized
 247 }
 248
 249 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 250         s.doVhostRequests(c, authzViaPOST)
 251 }
 252 func authzViaXHRPOST(r *http.Request, tok string) int {
 253         r.Method = "POST"
 254         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 255         r.Header.Add("Origin", "https://origin.example")
 256         r.Body = ioutil.NopCloser(strings.NewReader(
 257                 url.Values{
 258                         "api_token":   {tok},
 259                         "disposition": {"attachment"},
 260                 }.Encode()))
 261         return http.StatusUnauthorized
 262 }
 263
 264 // Try some combinations of {url, token} using the given authorization
 265 // mechanism, and verify the result is correct.
 266 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 267         for _, hostPath := range []string{
 268                 arvadostest.FooCollection + ".example.com/foo",
 269                 arvadostest.FooCollection + "--collections.example.com/foo",
 270                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 271                 arvadostest.FooCollectionPDH + ".example.com/foo",
 272                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 273                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 274         } {
 275                 c.Log("doRequests: ", hostPath)
 276                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 277         }
 278 }
 279
 280 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 281         for _, tok := range []string{
 282                 arvadostest.ActiveToken,
 283                 arvadostest.ActiveToken[:15],
 284                 arvadostest.SpectatorToken,
 285                 "bogus",
 286                 "",
 287         } {
 288                 u := mustParseURL("http://" + hostPath)
 289                 req := &http.Request{
 290                         Method:     "GET",
 291                         Host:       u.Host,
 292                         URL:        u,
 293                         RequestURI: u.RequestURI(),
 294                         Header:     http.Header{},
 295                 }
 296                 failCode := authz(req, tok)
 297                 req, resp := s.doReq(req)
 298                 code, body := resp.Code, resp.Body.String()
 299
 300                 // If the initial request had a (non-empty) token
 301                 // showing in the query string, we should have been
 302                 // redirected in order to hide it in a cookie.
 303                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 304
 305                 if tok == arvadostest.ActiveToken {
 306                         c.Check(code, check.Equals, http.StatusOK)
 307                         c.Check(body, check.Equals, "foo")
 308                 } else {
 309                         c.Check(code >= 400, check.Equals, true)
 310                         c.Check(code < 500, check.Equals, true)
 311                         if tok == arvadostest.SpectatorToken {
 312                                 // Valid token never offers to retry
 313                                 // with different credentials.
 314                                 c.Check(code, check.Equals, http.StatusNotFound)
 315                         } else {
 316                                 // Invalid token can ask to retry
 317                                 // depending on the authz method.
 318                                 c.Check(code, check.Equals, failCode)
 319                         }
 320                         if code == 404 {
 321                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 322                         } else {
 323                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 324                         }
 325                 }
 326         }
 327 }
 328
 329 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 330         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 331                 for _, port := range []string{"80", "443", "8000"} {
 332                         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 333                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 334                         req := &http.Request{
 335                                 Method:     "GET",
 336                                 Host:       u.Host,
 337                                 URL:        u,
 338                                 RequestURI: u.RequestURI(),
 339                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 340                         }
 341                         req, resp := s.doReq(req)
 342                         code, _ := resp.Code, resp.Body.String()
 343
 344                         if port == "8000" {
 345                                 c.Check(code, check.Equals, 401)
 346                         } else {
 347                                 c.Check(code, check.Equals, 200)
 348                         }
 349                 }
 350         }
 351 }
 352
 353 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 354         resp := httptest.NewRecorder()
 355         s.testServer.Handler.ServeHTTP(resp, req)
 356         if resp.Code != http.StatusSeeOther {
 357                 return req, resp
 358         }
 359         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 360         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 361         req = &http.Request{
 362                 Method:     "GET",
 363                 Host:       u.Host,
 364                 URL:        u,
 365                 RequestURI: u.RequestURI(),
 366                 Header:     http.Header{},
 367         }
 368         for _, c := range cookies {
 369                 req.AddCookie(c)
 370         }
 371         return s.doReq(req)
 372 }
 373
 374 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 375         s.testVhostRedirectTokenToCookie(c, "GET",
 376                 arvadostest.FooCollection+".example.com/foo",
 377                 "?api_token="+arvadostest.ActiveToken,
 378                 "",
 379                 "",
 380                 http.StatusOK,
 381                 "foo",
 382         )
 383 }
 384
 385 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 386         s.testVhostRedirectTokenToCookie(c, "GET",
 387                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 388                 "",
 389                 "",
 390                 "",
 391                 http.StatusOK,
 392                 "foo",
 393         )
 394 }
 395
 396 // Bad token in URL is 404 Not Found because it doesn't make sense to
 397 // retry the same URL with different authorization.
 398 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 399         s.testVhostRedirectTokenToCookie(c, "GET",
 400                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 401                 "",
 402                 "",
 403                 "",
 404                 http.StatusNotFound,
 405                 notFoundMessage+"\n",
 406         )
 407 }
 408
 409 // Bad token in a cookie (even if it got there via our own
 410 // query-string-to-cookie redirect) is, in principle, retryable at the
 411 // same URL so it's 401 Unauthorized.
 412 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 413         s.testVhostRedirectTokenToCookie(c, "GET",
 414                 arvadostest.FooCollection+".example.com/foo",
 415                 "?api_token=thisisabogustoken",
 416                 "",
 417                 "",
 418                 http.StatusUnauthorized,
 419                 unauthorizedMessage+"\n",
 420         )
 421 }
 422
 423 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 424         s.testVhostRedirectTokenToCookie(c, "GET",
 425                 "example.com/c="+arvadostest.FooCollection+"/foo",
 426                 "?api_token="+arvadostest.ActiveToken,
 427                 "",
 428                 "",
 429                 http.StatusBadRequest,
 430                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 431         )
 432 }
 433
 434 // If client requests an attachment by putting ?disposition=attachment
 435 // in the query string, and gets redirected, the redirect target
 436 // should respond with an attachment.
 437 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 438         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 439                 arvadostest.FooCollection+".example.com/foo",
 440                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 441                 "",
 442                 "",
 443                 http.StatusOK,
 444                 "foo",
 445         )
 446         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 447 }
 448
 449 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 450         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 451         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 452                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 453                 "?api_token="+arvadostest.ActiveToken,
 454                 "",
 455                 "",
 456                 http.StatusOK,
 457                 "foo",
 458         )
 459         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 460 }
 461
 462 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 463         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 464         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 465                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 466                 "?api_token="+arvadostest.ActiveToken,
 467                 "",
 468                 "",
 469                 http.StatusOK,
 470                 "waz",
 471         )
 472         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 473         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 474                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 475                 "?api_token="+arvadostest.ActiveToken,
 476                 "",
 477                 "",
 478                 http.StatusOK,
 479                 "waz",
 480         )
 481         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 482 }
 483
 484 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 485         s.testServer.Config.cluster.Collections.TrustAllContent = true
 486         s.testVhostRedirectTokenToCookie(c, "GET",
 487                 "example.com/c="+arvadostest.FooCollection+"/foo",
 488                 "?api_token="+arvadostest.ActiveToken,
 489                 "",
 490                 "",
 491                 http.StatusOK,
 492                 "foo",
 493         )
 494 }
 495
 496 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 497         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 498
 499         s.testVhostRedirectTokenToCookie(c, "GET",
 500                 "example.com/c="+arvadostest.FooCollection+"/foo",
 501                 "?api_token="+arvadostest.ActiveToken,
 502                 "",
 503                 "",
 504                 http.StatusBadRequest,
 505                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 506         )
 507
 508         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 509                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 510                 "?api_token="+arvadostest.ActiveToken,
 511                 "",
 512                 "",
 513                 http.StatusOK,
 514                 "foo",
 515         )
 516         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 517 }
 518
 519 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 520         s.testVhostRedirectTokenToCookie(c, "POST",
 521                 arvadostest.FooCollection+".example.com/foo",
 522                 "",
 523                 "application/x-www-form-urlencoded",
 524                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 525                 http.StatusOK,
 526                 "foo",
 527         )
 528 }
 529
 530 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 531         s.testVhostRedirectTokenToCookie(c, "POST",
 532                 arvadostest.FooCollection+".example.com/foo",
 533                 "",
 534                 "application/x-www-form-urlencoded",
 535                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 536                 http.StatusNotFound,
 537                 notFoundMessage+"\n",
 538         )
 539 }
 540
 541 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 542         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 543         s.testVhostRedirectTokenToCookie(c, "GET",
 544                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 545                 "",
 546                 "",
 547                 "",
 548                 http.StatusOK,
 549                 "Hello world\n",
 550         )
 551 }
 552
 553 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 554         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 555         s.testVhostRedirectTokenToCookie(c, "GET",
 556                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 557                 "",
 558                 "",
 559                 "",
 560                 http.StatusNotFound,
 561                 notFoundMessage+"\n",
 562         )
 563 }
 564
 565 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 566         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 567
 568         client := s.testServer.Config.Client
 569         client.AuthToken = arvadostest.ActiveToken
 570         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 571         c.Assert(err, check.IsNil)
 572         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 573         c.Assert(err, check.IsNil)
 574         f.Close()
 575         mtxt, err := fs.MarshalManifest(".")
 576         c.Assert(err, check.IsNil)
 577         var coll arvados.Collection
 578         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 579                 "collection": map[string]string{
 580                         "manifest_text": mtxt,
 581                 },
 582         })
 583         c.Assert(err, check.IsNil)
 584
 585         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 586         req := &http.Request{
 587                 Method:     "GET",
 588                 Host:       u.Host,
 589                 URL:        u,
 590                 RequestURI: u.RequestURI(),
 591                 Header: http.Header{
 592                         "Authorization": {"Bearer " + client.AuthToken},
 593                 },
 594         }
 595         resp := httptest.NewRecorder()
 596         s.testServer.Handler.ServeHTTP(resp, req)
 597         c.Check(resp.Code, check.Equals, http.StatusOK)
 598         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 599 }
 600
 601 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 602         arv := arvados.NewClientFromEnv()
 603         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 604         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 605         name := "foo/bar/baz"
 606         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 607         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 608
 609         client := s.testServer.Config.Client
 610         client.AuthToken = arvadostest.ActiveToken
 611         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 612         c.Assert(err, check.IsNil)
 613         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 614         c.Assert(err, check.IsNil)
 615         f.Close()
 616         mtxt, err := fs.MarshalManifest(".")
 617         c.Assert(err, check.IsNil)
 618         var coll arvados.Collection
 619         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 620                 "collection": map[string]string{
 621                         "manifest_text": mtxt,
 622                         "name":          name,
 623                         "owner_uuid":    arvadostest.AProjectUUID,
 624                 },
 625         })
 626         c.Assert(err, check.IsNil)
 627         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 628
 629         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 630         for tryURL, expectRegexp := range map[string]string{
 631                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 632                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 633         } {
 634                 u, _ := url.Parse(tryURL)
 635                 req := &http.Request{
 636                         Method:     "GET",
 637                         Host:       u.Host,
 638                         URL:        u,
 639                         RequestURI: u.RequestURI(),
 640                         Header: http.Header{
 641                                 "Authorization": {"Bearer " + client.AuthToken},
 642                         },
 643                 }
 644                 resp := httptest.NewRecorder()
 645                 s.testServer.Handler.ServeHTTP(resp, req)
 646                 c.Check(resp.Code, check.Equals, http.StatusOK)
 647                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 648         }
 649 }
 650
 651 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 652 // and disposition=attachment (telling us it's acceptable to respond
 653 // with content instead of a redirect) and an Origin header that gets
 654 // added automatically by the browser (telling us it's desirable to do
 655 // so).
 656 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 657         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 658         req := &http.Request{
 659                 Method:     "POST",
 660                 Host:       u.Host,
 661                 URL:        u,
 662                 RequestURI: u.RequestURI(),
 663                 Header: http.Header{
 664                         "Origin":       {"https://origin.example"},
 665                         "Content-Type": {"application/x-www-form-urlencoded"},
 666                 },
 667                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 668                         "api_token":   {arvadostest.ActiveToken},
 669                         "disposition": {"attachment"},
 670                 }.Encode())),
 671         }
 672         resp := httptest.NewRecorder()
 673         s.testServer.Handler.ServeHTTP(resp, req)
 674         c.Check(resp.Code, check.Equals, http.StatusOK)
 675         c.Check(resp.Body.String(), check.Equals, "foo")
 676         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 677
 678         // GET + Origin header is representative of both AJAX GET
 679         // requests and inline images via <IMG crossorigin="anonymous"
 680         // src="...">.
 681         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 682         req = &http.Request{
 683                 Method:     "GET",
 684                 Host:       u.Host,
 685                 URL:        u,
 686                 RequestURI: u.RequestURI(),
 687                 Header: http.Header{
 688                         "Origin": {"https://origin.example"},
 689                 },
 690         }
 691         resp = httptest.NewRecorder()
 692         s.testServer.Handler.ServeHTTP(resp, req)
 693         c.Check(resp.Code, check.Equals, http.StatusOK)
 694         c.Check(resp.Body.String(), check.Equals, "foo")
 695         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 696 }
 697
 698 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 699         u, _ := url.Parse(`http://` + hostPath + queryString)
 700         req := &http.Request{
 701                 Method:     method,
 702                 Host:       u.Host,
 703                 URL:        u,
 704                 RequestURI: u.RequestURI(),
 705                 Header:     http.Header{"Content-Type": {contentType}},
 706                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 707         }
 708
 709         resp := httptest.NewRecorder()
 710         defer func() {
 711                 c.Check(resp.Code, check.Equals, expectStatus)
 712                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 713         }()
 714
 715         s.testServer.Handler.ServeHTTP(resp, req)
 716         if resp.Code != http.StatusSeeOther {
 717                 return resp
 718         }
 719         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 720         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 721
 722         u, _ = u.Parse(resp.Header().Get("Location"))
 723         req = &http.Request{
 724                 Method:     "GET",
 725                 Host:       u.Host,
 726                 URL:        u,
 727                 RequestURI: u.RequestURI(),
 728                 Header:     http.Header{},
 729         }
 730         for _, c := range cookies {
 731                 req.AddCookie(c)
 732         }
 733
 734         resp = httptest.NewRecorder()
 735         s.testServer.Handler.ServeHTTP(resp, req)
 736         c.Check(resp.Header().Get("Location"), check.Equals, "")
 737         return resp
 738 }
 739
 740 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 741         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 742         s.testDirectoryListing(c)
 743 }
 744
 745 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 746         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 747         s.testDirectoryListing(c)
 748 }
 749
 750 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 751         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 752         authHeader := http.Header{
 753                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 754         }
 755         for _, trial := range []struct {
 756                 uri      string
 757                 header   http.Header
 758                 expect   []string
 759                 redirect string
 760                 cutDirs  int
 761         }{
 762                 {
 763                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 764                         header:  authHeader,
 765                         expect:  []string{"dir1/foo", "dir1/bar"},
 766                         cutDirs: 0,
 767                 },
 768                 {
 769                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 770                         header:  authHeader,
 771                         expect:  []string{"foo", "bar"},
 772                         cutDirs: 1,
 773                 },
 774                 {
 775                         // URLs of this form ignore authHeader, and
 776                         // FooAndBarFilesInDirUUID isn't public, so
 777                         // this returns 401.
 778                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 779                         header: authHeader,
 780                         expect: nil,
 781                 },
 782                 {
 783                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 784                         header:  authHeader,
 785                         expect:  []string{"dir1/"},
 786                         cutDirs: 3,
 787                 },
 788                 {
 789                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 790                         header:  authHeader,
 791                         expect:  []string{"bar"},
 792                         cutDirs: 4,
 793                 },
 794                 {
 795                         uri:     "download.example.com/",
 796                         header:  authHeader,
 797                         expect:  []string{"users/"},
 798                         cutDirs: 0,
 799                 },
 800                 {
 801                         uri:      "download.example.com/users",
 802                         header:   authHeader,
 803                         redirect: "/users/",
 804                         expect:   []string{"active/"},
 805                         cutDirs:  1,
 806                 },
 807                 {
 808                         uri:     "download.example.com/users/",
 809                         header:  authHeader,
 810                         expect:  []string{"active/"},
 811                         cutDirs: 1,
 812                 },
 813                 {
 814                         uri:      "download.example.com/users/active",
 815                         header:   authHeader,
 816                         redirect: "/users/active/",
 817                         expect:   []string{"foo_file_in_dir/"},
 818                         cutDirs:  2,
 819                 },
 820                 {
 821                         uri:     "download.example.com/users/active/",
 822                         header:  authHeader,
 823                         expect:  []string{"foo_file_in_dir/"},
 824                         cutDirs: 2,
 825                 },
 826                 {
 827                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 828                         header:  nil,
 829                         expect:  []string{"dir1/foo", "dir1/bar"},
 830                         cutDirs: 4,
 831                 },
 832                 {
 833                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 834                         header:  nil,
 835                         expect:  []string{"dir1/foo", "dir1/bar"},
 836                         cutDirs: 2,
 837                 },
 838                 {
 839                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 840                         header:  nil,
 841                         expect:  []string{"dir1/foo", "dir1/bar"},
 842                         cutDirs: 2,
 843                 },
 844                 {
 845                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 846                         header:  authHeader,
 847                         expect:  []string{"dir1/foo", "dir1/bar"},
 848                         cutDirs: 1,
 849                 },
 850                 {
 851                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 852                         header:   authHeader,
 853                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 854                         expect:   []string{"foo", "bar"},
 855                         cutDirs:  2,
 856                 },
 857                 {
 858                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 859                         header:  authHeader,
 860                         expect:  []string{"foo", "bar"},
 861                         cutDirs: 3,
 862                 },
 863                 {
 864                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 865                         header:   authHeader,
 866                         redirect: "/dir1/",
 867                         expect:   []string{"foo", "bar"},
 868                         cutDirs:  1,
 869                 },
 870                 {
 871                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 872                         header: authHeader,
 873                         expect: nil,
 874                 },
 875                 {
 876                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 877                         header:  authHeader,
 878                         expect:  []string{"waz"},
 879                         cutDirs: 1,
 880                 },
 881                 {
 882                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 883                         header:  authHeader,
 884                         expect:  []string{"waz"},
 885                         cutDirs: 2,
 886                 },
 887         } {
 888                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 889                 resp := httptest.NewRecorder()
 890                 u := mustParseURL("//" + trial.uri)
 891                 req := &http.Request{
 892                         Method:     "GET",
 893                         Host:       u.Host,
 894                         URL:        u,
 895                         RequestURI: u.RequestURI(),
 896                         Header:     copyHeader(trial.header),
 897                 }
 898                 s.testServer.Handler.ServeHTTP(resp, req)
 899                 var cookies []*http.Cookie
 900                 for resp.Code == http.StatusSeeOther {
 901                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 902                         req = &http.Request{
 903                                 Method:     "GET",
 904                                 Host:       u.Host,
 905                                 URL:        u,
 906                                 RequestURI: u.RequestURI(),
 907                                 Header:     copyHeader(trial.header),
 908                         }
 909                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 910                         for _, c := range cookies {
 911                                 req.AddCookie(c)
 912                         }
 913                         resp = httptest.NewRecorder()
 914                         s.testServer.Handler.ServeHTTP(resp, req)
 915                 }
 916                 if trial.redirect != "" {
 917                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 918                 }
 919                 if trial.expect == nil {
 920                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 921                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 922                         } else {
 923                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 924                         }
 925                 } else {
 926                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 927                         for _, e := range trial.expect {
 928                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 929                         }
 930                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 931                 }
 932
 933                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 934                 req = &http.Request{
 935                         Method:     "OPTIONS",
 936                         Host:       u.Host,
 937                         URL:        u,
 938                         RequestURI: u.RequestURI(),
 939                         Header:     copyHeader(trial.header),
 940                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 941                 }
 942                 resp = httptest.NewRecorder()
 943                 s.testServer.Handler.ServeHTTP(resp, req)
 944                 if trial.expect == nil {
 945                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 946                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 947                         } else {
 948                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 949                         }
 950                 } else {
 951                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 952                 }
 953
 954                 req = &http.Request{
 955                         Method:     "PROPFIND",
 956                         Host:       u.Host,
 957                         URL:        u,
 958                         RequestURI: u.RequestURI(),
 959                         Header:     copyHeader(trial.header),
 960                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 961                 }
 962                 resp = httptest.NewRecorder()
 963                 s.testServer.Handler.ServeHTTP(resp, req)
 964                 if trial.expect == nil {
 965                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 966                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 967                         } else {
 968                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 969                         }
 970                 } else {
 971                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 972                         for _, e := range trial.expect {
 973                                 if strings.HasSuffix(e, "/") {
 974                                         e = filepath.Join(u.Path, e) + "/"
 975                                 } else {
 976                                         e = filepath.Join(u.Path, e)
 977                                 }
 978                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 979                         }
 980                 }
 981         }
 982 }
 983
 984 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 985         arv := arvados.NewClientFromEnv()
 986         var newCollection arvados.Collection
 987         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 988                 "collection": map[string]string{
 989                         "owner_uuid":    arvadostest.ActiveUserUUID,
 990                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 991                         "name":          "keep-web test collection",
 992                 },
 993                 "ensure_unique_name": true,
 994         })
 995         c.Assert(err, check.IsNil)
 996         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 997
 998         var updated arvados.Collection
 999         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1000                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1001                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1002                 req := &http.Request{
1003                         Method:     "DELETE",
1004                         Host:       u.Host,
1005                         URL:        u,
1006                         RequestURI: u.RequestURI(),
1007                         Header: http.Header{
1008                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1009                         },
1010                 }
1011                 resp := httptest.NewRecorder()
1012                 s.testServer.Handler.ServeHTTP(resp, req)
1013                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1014
1015                 updated = arvados.Collection{}
1016                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1017                 c.Check(err, check.IsNil)
1018                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1019                 c.Logf("updated manifest_text %q", updated.ManifestText)
1020         }
1021         c.Check(updated.ManifestText, check.Equals, "")
1022 }
1023
1024 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
1025         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
1026         authHeader := http.Header{
1027                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
1028         }
1029
1030         resp := httptest.NewRecorder()
1031         u := mustParseURL("http://download.example.com/_health/ping")
1032         req := &http.Request{
1033                 Method:     "GET",
1034                 Host:       u.Host,
1035                 URL:        u,
1036                 RequestURI: u.RequestURI(),
1037                 Header:     authHeader,
1038         }
1039         s.testServer.Handler.ServeHTTP(resp, req)
1040
1041         c.Check(resp.Code, check.Equals, http.StatusOK)
1042         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
1043 }
1044
1045 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1046         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1047
1048         client := s.testServer.Config.Client
1049         client.AuthToken = arvadostest.ActiveToken
1050         arv, err := arvadosclient.New(&client)
1051         c.Assert(err, check.Equals, nil)
1052         kc, err := keepclient.MakeKeepClient(arv)
1053         c.Assert(err, check.Equals, nil)
1054
1055         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
1056         c.Assert(err, check.IsNil)
1057
1058         trials := []struct {
1059                 filename    string
1060                 content     string
1061                 contentType string
1062         }{
1063                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1064                 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
1065                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1066                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1067                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1068         }
1069         for _, trial := range trials {
1070                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1071                 c.Assert(err, check.IsNil)
1072                 _, err = f.Write([]byte(trial.content))
1073                 c.Assert(err, check.IsNil)
1074                 c.Assert(f.Close(), check.IsNil)
1075         }
1076         mtxt, err := fs.MarshalManifest(".")
1077         c.Assert(err, check.IsNil)
1078         var coll arvados.Collection
1079         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1080                 "collection": map[string]string{
1081                         "manifest_text": mtxt,
1082                 },
1083         })
1084         c.Assert(err, check.IsNil)
1085
1086         for _, trial := range trials {
1087                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1088                 req := &http.Request{
1089                         Method:     "GET",
1090                         Host:       u.Host,
1091                         URL:        u,
1092                         RequestURI: u.RequestURI(),
1093                         Header: http.Header{
1094                                 "Authorization": {"Bearer " + client.AuthToken},
1095                         },
1096                 }
1097                 resp := httptest.NewRecorder()
1098                 s.testServer.Handler.ServeHTTP(resp, req)
1099                 c.Check(resp.Code, check.Equals, http.StatusOK)
1100                 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
1101                 c.Check(resp.Body.String(), check.Equals, trial.content)
1102         }
1103 }
1104
1105 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1106         s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1107         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1108         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1109         req := &http.Request{
1110                 Method:     "GET",
1111                 Host:       u.Host,
1112                 URL:        u,
1113                 RequestURI: u.RequestURI(),
1114         }
1115         resp := httptest.NewRecorder()
1116         s.testServer.Handler.ServeHTTP(resp, req)
1117         c.Check(resp.Code, check.Equals, http.StatusOK)
1118         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1119 }
1120
1121 // Writing to a collection shouldn't affect its entry in the
1122 // PDH-to-manifest cache.
1123 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1124         arv, err := arvadosclient.MakeArvadosClient()
1125         c.Assert(err, check.Equals, nil)
1126         arv.ApiToken = arvadostest.ActiveToken
1127
1128         u := mustParseURL("http://x.example/testfile")
1129         req := &http.Request{
1130                 Method:     "GET",
1131                 Host:       u.Host,
1132                 URL:        u,
1133                 RequestURI: u.RequestURI(),
1134                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1135         }
1136
1137         checkWithID := func(id string, status int) {
1138                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1139                 req.Host = req.URL.Host
1140                 resp := httptest.NewRecorder()
1141                 s.testServer.Handler.ServeHTTP(resp, req)
1142                 c.Check(resp.Code, check.Equals, status)
1143         }
1144
1145         var colls [2]arvados.Collection
1146         for i := range colls {
1147                 err := arv.Create("collections",
1148                         map[string]interface{}{
1149                                 "ensure_unique_name": true,
1150                                 "collection": map[string]interface{}{
1151                                         "name": "test collection",
1152                                 },
1153                         }, &colls[i])
1154                 c.Assert(err, check.Equals, nil)
1155         }
1156
1157         // Populate cache with empty collection
1158         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1159
1160         // write a file to colls[0]
1161         reqPut := *req
1162         reqPut.Method = "PUT"
1163         reqPut.URL.Host = colls[0].UUID + ".example"
1164         reqPut.Host = req.URL.Host
1165         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1166         resp := httptest.NewRecorder()
1167         s.testServer.Handler.ServeHTTP(resp, &reqPut)
1168         c.Check(resp.Code, check.Equals, http.StatusCreated)
1169
1170         // new file should not appear in colls[1]
1171         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1172         checkWithID(colls[1].UUID, http.StatusNotFound)
1173
1174         checkWithID(colls[0].UUID, http.StatusOK)
1175 }
1176
1177 func copyHeader(h http.Header) http.Header {
1178         hc := http.Header{}
1179         for k, v := range h {
1180                 hc[k] = append([]string(nil), v...)
1181         }
1182         return hc
1183 }