services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "html"
  12         "io"
  13         "io/ioutil"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "path/filepath"
  19         "regexp"
  20         "strings"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/sirupsen/logrus"
  31         check "gopkg.in/check.v1"
  32 )
  33
  34 var _ = check.Suite(&UnitSuite{})
  35
  36 func init() {
  37         arvados.DebugLocksPanicMode = true
  38 }
  39
  40 type UnitSuite struct {
  41         Config *arvados.Config
  42 }
  43
  44 func (s *UnitSuite) SetUpTest(c *check.C) {
  45         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
  46         ldr.Path = "-"
  47         cfg, err := ldr.Load()
  48         c.Assert(err, check.IsNil)
  49         s.Config = cfg
  50 }
  51
  52 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
  53         h := handler{Config: newConfig(s.Config)}
  54         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
  55         req := &http.Request{
  56                 Method:     "OPTIONS",
  57                 Host:       u.Host,
  58                 URL:        u,
  59                 RequestURI: u.RequestURI(),
  60                 Header: http.Header{
  61                         "Origin":                        {"https://workbench.example"},
  62                         "Access-Control-Request-Method": {"POST"},
  63                 },
  64         }
  65
  66         // Check preflight for an allowed request
  67         resp := httptest.NewRecorder()
  68         h.ServeHTTP(resp, req)
  69         c.Check(resp.Code, check.Equals, http.StatusOK)
  70         c.Check(resp.Body.String(), check.Equals, "")
  71         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
  72         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
  73         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout")
  74
  75         // Check preflight for a disallowed request
  76         resp = httptest.NewRecorder()
  77         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
  78         h.ServeHTTP(resp, req)
  79         c.Check(resp.Body.String(), check.Equals, "")
  80         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
  81 }
  82
  83 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
  84         for _, trial := range []struct {
  85                 dataExists    bool
  86                 sendIMSHeader bool
  87                 expectStatus  int
  88                 logRegexp     string
  89         }{
  90                 // If we return no content due to a Keep read error,
  91                 // we should emit a log message.
  92                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
  93
  94                 // If we return no content because the client sent an
  95                 // If-Modified-Since header, our response should be
  96                 // 304, and we should not emit a log message.
  97                 {true, true, http.StatusNotModified, ``},
  98         } {
  99                 c.Logf("trial: %+v", trial)
 100                 arvadostest.StartKeep(2, true)
 101                 if trial.dataExists {
 102                         arv, err := arvadosclient.MakeArvadosClient()
 103                         c.Assert(err, check.IsNil)
 104                         arv.ApiToken = arvadostest.ActiveToken
 105                         kc, err := keepclient.MakeKeepClient(arv)
 106                         c.Assert(err, check.IsNil)
 107                         _, _, err = kc.PutB([]byte("foo"))
 108                         c.Assert(err, check.IsNil)
 109                 }
 110
 111                 h := handler{Config: newConfig(s.Config)}
 112                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 113                 req := &http.Request{
 114                         Method:     "GET",
 115                         Host:       u.Host,
 116                         URL:        u,
 117                         RequestURI: u.RequestURI(),
 118                         Header: http.Header{
 119                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 120                         },
 121                 }
 122                 if trial.sendIMSHeader {
 123                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 124                 }
 125
 126                 var logbuf bytes.Buffer
 127                 logger := logrus.New()
 128                 logger.Out = &logbuf
 129                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 130
 131                 resp := httptest.NewRecorder()
 132                 h.ServeHTTP(resp, req)
 133                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 134                 c.Check(resp.Body.String(), check.Equals, "")
 135
 136                 c.Log(logbuf.String())
 137                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 138         }
 139 }
 140
 141 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 142         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 143         token := arvadostest.ActiveToken
 144         for _, trial := range []string{
 145                 "http://keep-web/c=" + bogusID + "/foo",
 146                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 147                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 148                 "http://keep-web/collections/" + bogusID + "/foo",
 149                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 150                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 151         } {
 152                 c.Log(trial)
 153                 u := mustParseURL(trial)
 154                 req := &http.Request{
 155                         Method:     "GET",
 156                         Host:       u.Host,
 157                         URL:        u,
 158                         RequestURI: u.RequestURI(),
 159                 }
 160                 resp := httptest.NewRecorder()
 161                 cfg := newConfig(s.Config)
 162                 cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 163                 h := handler{Config: cfg}
 164                 h.ServeHTTP(resp, req)
 165                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 166         }
 167 }
 168
 169 func mustParseURL(s string) *url.URL {
 170         r, err := url.Parse(s)
 171         if err != nil {
 172                 panic("parse URL: " + s)
 173         }
 174         return r
 175 }
 176
 177 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 178         for _, testURL := range []string{
 179                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 180                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 181         } {
 182                 resp := httptest.NewRecorder()
 183                 u := mustParseURL(testURL)
 184                 req := &http.Request{
 185                         Method:     "GET",
 186                         URL:        u,
 187                         RequestURI: u.RequestURI(),
 188                 }
 189                 s.testServer.Handler.ServeHTTP(resp, req)
 190                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 191                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 192         }
 193 }
 194
 195 // An authorizer modifies an HTTP request to make use of the given
 196 // token -- by adding it to a header, cookie, query param, or whatever
 197 // -- and returns the HTTP status code we should expect from keep-web if
 198 // the token is invalid.
 199 type authorizer func(*http.Request, string) int
 200
 201 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 202         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 203 }
 204 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 205         r.Header.Add("Authorization", "Bearer "+tok)
 206         return http.StatusUnauthorized
 207 }
 208 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 209         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 210 }
 211 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 212         r.Header.Add("Authorization", "Bearer "+tok)
 213         return http.StatusUnauthorized
 214 }
 215
 216 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 217         s.doVhostRequests(c, authzViaCookieValue)
 218 }
 219 func authzViaCookieValue(r *http.Request, tok string) int {
 220         r.AddCookie(&http.Cookie{
 221                 Name:  "arvados_api_token",
 222                 Value: auth.EncodeTokenCookie([]byte(tok)),
 223         })
 224         return http.StatusUnauthorized
 225 }
 226
 227 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 228         s.doVhostRequests(c, authzViaPath)
 229 }
 230 func authzViaPath(r *http.Request, tok string) int {
 231         r.URL.Path = "/t=" + tok + r.URL.Path
 232         return http.StatusNotFound
 233 }
 234
 235 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 236         s.doVhostRequests(c, authzViaQueryString)
 237 }
 238 func authzViaQueryString(r *http.Request, tok string) int {
 239         r.URL.RawQuery = "api_token=" + tok
 240         return http.StatusUnauthorized
 241 }
 242
 243 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 244         s.doVhostRequests(c, authzViaPOST)
 245 }
 246 func authzViaPOST(r *http.Request, tok string) int {
 247         r.Method = "POST"
 248         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 249         r.Body = ioutil.NopCloser(strings.NewReader(
 250                 url.Values{"api_token": {tok}}.Encode()))
 251         return http.StatusUnauthorized
 252 }
 253
 254 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 255         s.doVhostRequests(c, authzViaPOST)
 256 }
 257 func authzViaXHRPOST(r *http.Request, tok string) int {
 258         r.Method = "POST"
 259         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 260         r.Header.Add("Origin", "https://origin.example")
 261         r.Body = ioutil.NopCloser(strings.NewReader(
 262                 url.Values{
 263                         "api_token":   {tok},
 264                         "disposition": {"attachment"},
 265                 }.Encode()))
 266         return http.StatusUnauthorized
 267 }
 268
 269 // Try some combinations of {url, token} using the given authorization
 270 // mechanism, and verify the result is correct.
 271 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 272         for _, hostPath := range []string{
 273                 arvadostest.FooCollection + ".example.com/foo",
 274                 arvadostest.FooCollection + "--collections.example.com/foo",
 275                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 276                 arvadostest.FooCollectionPDH + ".example.com/foo",
 277                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 278                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 279         } {
 280                 c.Log("doRequests: ", hostPath)
 281                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 282         }
 283 }
 284
 285 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 286         for _, tok := range []string{
 287                 arvadostest.ActiveToken,
 288                 arvadostest.ActiveToken[:15],
 289                 arvadostest.SpectatorToken,
 290                 "bogus",
 291                 "",
 292         } {
 293                 u := mustParseURL("http://" + hostPath)
 294                 req := &http.Request{
 295                         Method:     "GET",
 296                         Host:       u.Host,
 297                         URL:        u,
 298                         RequestURI: u.RequestURI(),
 299                         Header:     http.Header{},
 300                 }
 301                 failCode := authz(req, tok)
 302                 req, resp := s.doReq(req)
 303                 code, body := resp.Code, resp.Body.String()
 304
 305                 // If the initial request had a (non-empty) token
 306                 // showing in the query string, we should have been
 307                 // redirected in order to hide it in a cookie.
 308                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 309
 310                 if tok == arvadostest.ActiveToken {
 311                         c.Check(code, check.Equals, http.StatusOK)
 312                         c.Check(body, check.Equals, "foo")
 313                 } else {
 314                         c.Check(code >= 400, check.Equals, true)
 315                         c.Check(code < 500, check.Equals, true)
 316                         if tok == arvadostest.SpectatorToken {
 317                                 // Valid token never offers to retry
 318                                 // with different credentials.
 319                                 c.Check(code, check.Equals, http.StatusNotFound)
 320                         } else {
 321                                 // Invalid token can ask to retry
 322                                 // depending on the authz method.
 323                                 c.Check(code, check.Equals, failCode)
 324                         }
 325                         if code == 404 {
 326                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 327                         } else {
 328                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 329                         }
 330                 }
 331         }
 332 }
 333
 334 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 335         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 336                 for _, port := range []string{"80", "443", "8000"} {
 337                         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 338                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 339                         req := &http.Request{
 340                                 Method:     "GET",
 341                                 Host:       u.Host,
 342                                 URL:        u,
 343                                 RequestURI: u.RequestURI(),
 344                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 345                         }
 346                         req, resp := s.doReq(req)
 347                         code, _ := resp.Code, resp.Body.String()
 348
 349                         if port == "8000" {
 350                                 c.Check(code, check.Equals, 401)
 351                         } else {
 352                                 c.Check(code, check.Equals, 200)
 353                         }
 354                 }
 355         }
 356 }
 357
 358 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 359         resp := httptest.NewRecorder()
 360         s.testServer.Handler.ServeHTTP(resp, req)
 361         if resp.Code != http.StatusSeeOther {
 362                 return req, resp
 363         }
 364         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 365         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 366         req = &http.Request{
 367                 Method:     "GET",
 368                 Host:       u.Host,
 369                 URL:        u,
 370                 RequestURI: u.RequestURI(),
 371                 Header:     http.Header{},
 372         }
 373         for _, c := range cookies {
 374                 req.AddCookie(c)
 375         }
 376         return s.doReq(req)
 377 }
 378
 379 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 380         s.testVhostRedirectTokenToCookie(c, "GET",
 381                 arvadostest.FooCollection+".example.com/foo",
 382                 "?api_token="+arvadostest.ActiveToken,
 383                 "",
 384                 "",
 385                 http.StatusOK,
 386                 "foo",
 387         )
 388 }
 389
 390 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 391         s.testVhostRedirectTokenToCookie(c, "GET",
 392                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 393                 "",
 394                 "",
 395                 "",
 396                 http.StatusOK,
 397                 "foo",
 398         )
 399 }
 400
 401 // Bad token in URL is 404 Not Found because it doesn't make sense to
 402 // retry the same URL with different authorization.
 403 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 404         s.testVhostRedirectTokenToCookie(c, "GET",
 405                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 406                 "",
 407                 "",
 408                 "",
 409                 http.StatusNotFound,
 410                 notFoundMessage+"\n",
 411         )
 412 }
 413
 414 // Bad token in a cookie (even if it got there via our own
 415 // query-string-to-cookie redirect) is, in principle, retryable at the
 416 // same URL so it's 401 Unauthorized.
 417 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 418         s.testVhostRedirectTokenToCookie(c, "GET",
 419                 arvadostest.FooCollection+".example.com/foo",
 420                 "?api_token=thisisabogustoken",
 421                 "",
 422                 "",
 423                 http.StatusUnauthorized,
 424                 unauthorizedMessage+"\n",
 425         )
 426 }
 427
 428 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
 429         s.testVhostRedirectTokenToCookie(c, "GET",
 430                 "example.com/c="+arvadostest.FooCollection+"/foo",
 431                 "?api_token="+arvadostest.ActiveToken,
 432                 "",
 433                 "",
 434                 http.StatusBadRequest,
 435                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 436         )
 437 }
 438
 439 // If client requests an attachment by putting ?disposition=attachment
 440 // in the query string, and gets redirected, the redirect target
 441 // should respond with an attachment.
 442 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
 443         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 444                 arvadostest.FooCollection+".example.com/foo",
 445                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
 446                 "",
 447                 "",
 448                 http.StatusOK,
 449                 "foo",
 450         )
 451         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 452 }
 453
 454 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 455         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 456         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 457                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
 458                 "?api_token="+arvadostest.ActiveToken,
 459                 "",
 460                 "",
 461                 http.StatusOK,
 462                 "foo",
 463         )
 464         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 465 }
 466
 467 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 468         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 469         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 470                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
 471                 "?api_token="+arvadostest.ActiveToken,
 472                 "",
 473                 "",
 474                 http.StatusOK,
 475                 "waz",
 476         )
 477         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 478         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 479                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
 480                 "?api_token="+arvadostest.ActiveToken,
 481                 "",
 482                 "",
 483                 http.StatusOK,
 484                 "waz",
 485         )
 486         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
 487 }
 488
 489 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
 490         s.testServer.Config.cluster.Collections.TrustAllContent = true
 491         s.testVhostRedirectTokenToCookie(c, "GET",
 492                 "example.com/c="+arvadostest.FooCollection+"/foo",
 493                 "?api_token="+arvadostest.ActiveToken,
 494                 "",
 495                 "",
 496                 http.StatusOK,
 497                 "foo",
 498         )
 499 }
 500
 501 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
 502         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 503
 504         s.testVhostRedirectTokenToCookie(c, "GET",
 505                 "example.com/c="+arvadostest.FooCollection+"/foo",
 506                 "?api_token="+arvadostest.ActiveToken,
 507                 "",
 508                 "",
 509                 http.StatusBadRequest,
 510                 "cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n",
 511         )
 512
 513         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 514                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
 515                 "?api_token="+arvadostest.ActiveToken,
 516                 "",
 517                 "",
 518                 http.StatusOK,
 519                 "foo",
 520         )
 521         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
 522 }
 523
 524 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
 525         s.testVhostRedirectTokenToCookie(c, "POST",
 526                 arvadostest.FooCollection+".example.com/foo",
 527                 "",
 528                 "application/x-www-form-urlencoded",
 529                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
 530                 http.StatusOK,
 531                 "foo",
 532         )
 533 }
 534
 535 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
 536         s.testVhostRedirectTokenToCookie(c, "POST",
 537                 arvadostest.FooCollection+".example.com/foo",
 538                 "",
 539                 "application/x-www-form-urlencoded",
 540                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
 541                 http.StatusNotFound,
 542                 notFoundMessage+"\n",
 543         )
 544 }
 545
 546 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 547         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 548         s.testVhostRedirectTokenToCookie(c, "GET",
 549                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 550                 "",
 551                 "",
 552                 "",
 553                 http.StatusOK,
 554                 "Hello world\n",
 555         )
 556 }
 557
 558 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 559         s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
 560         s.testVhostRedirectTokenToCookie(c, "GET",
 561                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
 562                 "",
 563                 "",
 564                 "",
 565                 http.StatusNotFound,
 566                 notFoundMessage+"\n",
 567         )
 568 }
 569
 570 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
 571         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 572
 573         client := s.testServer.Config.Client
 574         client.AuthToken = arvadostest.ActiveToken
 575         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 576         c.Assert(err, check.IsNil)
 577         f, err := fs.OpenFile("https:\\\"odd' path chars", os.O_CREATE, 0777)
 578         c.Assert(err, check.IsNil)
 579         f.Close()
 580         mtxt, err := fs.MarshalManifest(".")
 581         c.Assert(err, check.IsNil)
 582         var coll arvados.Collection
 583         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 584                 "collection": map[string]string{
 585                         "manifest_text": mtxt,
 586                 },
 587         })
 588         c.Assert(err, check.IsNil)
 589
 590         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
 591         req := &http.Request{
 592                 Method:     "GET",
 593                 Host:       u.Host,
 594                 URL:        u,
 595                 RequestURI: u.RequestURI(),
 596                 Header: http.Header{
 597                         "Authorization": {"Bearer " + client.AuthToken},
 598                 },
 599         }
 600         resp := httptest.NewRecorder()
 601         s.testServer.Handler.ServeHTTP(resp, req)
 602         c.Check(resp.Code, check.Equals, http.StatusOK)
 603         c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./https:%5c%22odd%27%20path%20chars"\S+https:\\&#34;odd&#39; path chars.*`)
 604 }
 605
 606 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
 607         arv := arvados.NewClientFromEnv()
 608         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 609         s.testServer.Config.cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
 610         name := "foo/bar/baz"
 611         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
 612         nameShownEscaped := strings.Replace(name, "/", "%7bSOLIDUS%7d", -1)
 613
 614         client := s.testServer.Config.Client
 615         client.AuthToken = arvadostest.ActiveToken
 616         fs, err := (&arvados.Collection{}).FileSystem(&client, nil)
 617         c.Assert(err, check.IsNil)
 618         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
 619         c.Assert(err, check.IsNil)
 620         f.Close()
 621         mtxt, err := fs.MarshalManifest(".")
 622         c.Assert(err, check.IsNil)
 623         var coll arvados.Collection
 624         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 625                 "collection": map[string]string{
 626                         "manifest_text": mtxt,
 627                         "name":          name,
 628                         "owner_uuid":    arvadostest.AProjectUUID,
 629                 },
 630         })
 631         c.Assert(err, check.IsNil)
 632         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 633
 634         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
 635         for tryURL, expectRegexp := range map[string]string{
 636                 base:                          `(?ms).*href="./` + nameShownEscaped + `/"\S+` + nameShown + `.*`,
 637                 base + nameShownEscaped + "/": `(?ms).*href="./filename"\S+filename.*`,
 638         } {
 639                 u, _ := url.Parse(tryURL)
 640                 req := &http.Request{
 641                         Method:     "GET",
 642                         Host:       u.Host,
 643                         URL:        u,
 644                         RequestURI: u.RequestURI(),
 645                         Header: http.Header{
 646                                 "Authorization": {"Bearer " + client.AuthToken},
 647                         },
 648                 }
 649                 resp := httptest.NewRecorder()
 650                 s.testServer.Handler.ServeHTTP(resp, req)
 651                 c.Check(resp.Code, check.Equals, http.StatusOK)
 652                 c.Check(resp.Body.String(), check.Matches, expectRegexp)
 653         }
 654 }
 655
 656 // XHRs can't follow redirect-with-cookie so they rely on method=POST
 657 // and disposition=attachment (telling us it's acceptable to respond
 658 // with content instead of a redirect) and an Origin header that gets
 659 // added automatically by the browser (telling us it's desirable to do
 660 // so).
 661 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
 662         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
 663         req := &http.Request{
 664                 Method:     "POST",
 665                 Host:       u.Host,
 666                 URL:        u,
 667                 RequestURI: u.RequestURI(),
 668                 Header: http.Header{
 669                         "Origin":       {"https://origin.example"},
 670                         "Content-Type": {"application/x-www-form-urlencoded"},
 671                 },
 672                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
 673                         "api_token":   {arvadostest.ActiveToken},
 674                         "disposition": {"attachment"},
 675                 }.Encode())),
 676         }
 677         resp := httptest.NewRecorder()
 678         s.testServer.Handler.ServeHTTP(resp, req)
 679         c.Check(resp.Code, check.Equals, http.StatusOK)
 680         c.Check(resp.Body.String(), check.Equals, "foo")
 681         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 682
 683         // GET + Origin header is representative of both AJAX GET
 684         // requests and inline images via <IMG crossorigin="anonymous"
 685         // src="...">.
 686         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
 687         req = &http.Request{
 688                 Method:     "GET",
 689                 Host:       u.Host,
 690                 URL:        u,
 691                 RequestURI: u.RequestURI(),
 692                 Header: http.Header{
 693                         "Origin": {"https://origin.example"},
 694                 },
 695         }
 696         resp = httptest.NewRecorder()
 697         s.testServer.Handler.ServeHTTP(resp, req)
 698         c.Check(resp.Code, check.Equals, http.StatusOK)
 699         c.Check(resp.Body.String(), check.Equals, "foo")
 700         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 701 }
 702
 703 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
 704         u, _ := url.Parse(`http://` + hostPath + queryString)
 705         req := &http.Request{
 706                 Method:     method,
 707                 Host:       u.Host,
 708                 URL:        u,
 709                 RequestURI: u.RequestURI(),
 710                 Header:     http.Header{"Content-Type": {contentType}},
 711                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
 712         }
 713
 714         resp := httptest.NewRecorder()
 715         defer func() {
 716                 c.Check(resp.Code, check.Equals, expectStatus)
 717                 c.Check(resp.Body.String(), check.Equals, expectRespBody)
 718         }()
 719
 720         s.testServer.Handler.ServeHTTP(resp, req)
 721         if resp.Code != http.StatusSeeOther {
 722                 return resp
 723         }
 724         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
 725         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 726
 727         u, _ = u.Parse(resp.Header().Get("Location"))
 728         req = &http.Request{
 729                 Method:     "GET",
 730                 Host:       u.Host,
 731                 URL:        u,
 732                 RequestURI: u.RequestURI(),
 733                 Header:     http.Header{},
 734         }
 735         for _, c := range cookies {
 736                 req.AddCookie(c)
 737         }
 738
 739         resp = httptest.NewRecorder()
 740         s.testServer.Handler.ServeHTTP(resp, req)
 741         c.Check(resp.Header().Get("Location"), check.Equals, "")
 742         return resp
 743 }
 744
 745 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
 746         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 747         s.testDirectoryListing(c)
 748 }
 749
 750 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
 751         s.testServer.Config.cluster.Users.AnonymousUserToken = ""
 752         s.testDirectoryListing(c)
 753 }
 754
 755 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
 756         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 757         authHeader := http.Header{
 758                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
 759         }
 760         for _, trial := range []struct {
 761                 uri      string
 762                 header   http.Header
 763                 expect   []string
 764                 redirect string
 765                 cutDirs  int
 766         }{
 767                 {
 768                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
 769                         header:  authHeader,
 770                         expect:  []string{"dir1/foo", "dir1/bar"},
 771                         cutDirs: 0,
 772                 },
 773                 {
 774                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
 775                         header:  authHeader,
 776                         expect:  []string{"foo", "bar"},
 777                         cutDirs: 1,
 778                 },
 779                 {
 780                         // URLs of this form ignore authHeader, and
 781                         // FooAndBarFilesInDirUUID isn't public, so
 782                         // this returns 401.
 783                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
 784                         header: authHeader,
 785                         expect: nil,
 786                 },
 787                 {
 788                         uri:     "download.example.com/users/active/foo_file_in_dir/",
 789                         header:  authHeader,
 790                         expect:  []string{"dir1/"},
 791                         cutDirs: 3,
 792                 },
 793                 {
 794                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
 795                         header:  authHeader,
 796                         expect:  []string{"bar"},
 797                         cutDirs: 4,
 798                 },
 799                 {
 800                         uri:     "download.example.com/",
 801                         header:  authHeader,
 802                         expect:  []string{"users/"},
 803                         cutDirs: 0,
 804                 },
 805                 {
 806                         uri:      "download.example.com/users",
 807                         header:   authHeader,
 808                         redirect: "/users/",
 809                         expect:   []string{"active/"},
 810                         cutDirs:  1,
 811                 },
 812                 {
 813                         uri:     "download.example.com/users/",
 814                         header:  authHeader,
 815                         expect:  []string{"active/"},
 816                         cutDirs: 1,
 817                 },
 818                 {
 819                         uri:      "download.example.com/users/active",
 820                         header:   authHeader,
 821                         redirect: "/users/active/",
 822                         expect:   []string{"foo_file_in_dir/"},
 823                         cutDirs:  2,
 824                 },
 825                 {
 826                         uri:     "download.example.com/users/active/",
 827                         header:  authHeader,
 828                         expect:  []string{"foo_file_in_dir/"},
 829                         cutDirs: 2,
 830                 },
 831                 {
 832                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
 833                         header:  nil,
 834                         expect:  []string{"dir1/foo", "dir1/bar"},
 835                         cutDirs: 4,
 836                 },
 837                 {
 838                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
 839                         header:  nil,
 840                         expect:  []string{"dir1/foo", "dir1/bar"},
 841                         cutDirs: 2,
 842                 },
 843                 {
 844                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
 845                         header:  nil,
 846                         expect:  []string{"dir1/foo", "dir1/bar"},
 847                         cutDirs: 2,
 848                 },
 849                 {
 850                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
 851                         header:  authHeader,
 852                         expect:  []string{"dir1/foo", "dir1/bar"},
 853                         cutDirs: 1,
 854                 },
 855                 {
 856                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
 857                         header:   authHeader,
 858                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
 859                         expect:   []string{"foo", "bar"},
 860                         cutDirs:  2,
 861                 },
 862                 {
 863                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
 864                         header:  authHeader,
 865                         expect:  []string{"foo", "bar"},
 866                         cutDirs: 3,
 867                 },
 868                 {
 869                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
 870                         header:   authHeader,
 871                         redirect: "/dir1/",
 872                         expect:   []string{"foo", "bar"},
 873                         cutDirs:  1,
 874                 },
 875                 {
 876                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
 877                         header: authHeader,
 878                         expect: nil,
 879                 },
 880                 {
 881                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
 882                         header:  authHeader,
 883                         expect:  []string{"waz"},
 884                         cutDirs: 1,
 885                 },
 886                 {
 887                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
 888                         header:  authHeader,
 889                         expect:  []string{"waz"},
 890                         cutDirs: 2,
 891                 },
 892         } {
 893                 comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
 894                 resp := httptest.NewRecorder()
 895                 u := mustParseURL("//" + trial.uri)
 896                 req := &http.Request{
 897                         Method:     "GET",
 898                         Host:       u.Host,
 899                         URL:        u,
 900                         RequestURI: u.RequestURI(),
 901                         Header:     copyHeader(trial.header),
 902                 }
 903                 s.testServer.Handler.ServeHTTP(resp, req)
 904                 var cookies []*http.Cookie
 905                 for resp.Code == http.StatusSeeOther {
 906                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 907                         req = &http.Request{
 908                                 Method:     "GET",
 909                                 Host:       u.Host,
 910                                 URL:        u,
 911                                 RequestURI: u.RequestURI(),
 912                                 Header:     copyHeader(trial.header),
 913                         }
 914                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
 915                         for _, c := range cookies {
 916                                 req.AddCookie(c)
 917                         }
 918                         resp = httptest.NewRecorder()
 919                         s.testServer.Handler.ServeHTTP(resp, req)
 920                 }
 921                 if trial.redirect != "" {
 922                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
 923                 }
 924                 if trial.expect == nil {
 925                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 926                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 927                         } else {
 928                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 929                         }
 930                 } else {
 931                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 932                         for _, e := range trial.expect {
 933                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
 934                         }
 935                         c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
 936                 }
 937
 938                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
 939                 req = &http.Request{
 940                         Method:     "OPTIONS",
 941                         Host:       u.Host,
 942                         URL:        u,
 943                         RequestURI: u.RequestURI(),
 944                         Header:     copyHeader(trial.header),
 945                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 946                 }
 947                 resp = httptest.NewRecorder()
 948                 s.testServer.Handler.ServeHTTP(resp, req)
 949                 if trial.expect == nil {
 950                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 951                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 952                         } else {
 953                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 954                         }
 955                 } else {
 956                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
 957                 }
 958
 959                 req = &http.Request{
 960                         Method:     "PROPFIND",
 961                         Host:       u.Host,
 962                         URL:        u,
 963                         RequestURI: u.RequestURI(),
 964                         Header:     copyHeader(trial.header),
 965                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
 966                 }
 967                 resp = httptest.NewRecorder()
 968                 s.testServer.Handler.ServeHTTP(resp, req)
 969                 if trial.expect == nil {
 970                         if s.testServer.Config.cluster.Users.AnonymousUserToken == "" {
 971                                 c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
 972                         } else {
 973                                 c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
 974                         }
 975                 } else {
 976                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
 977                         for _, e := range trial.expect {
 978                                 if strings.HasSuffix(e, "/") {
 979                                         e = filepath.Join(u.Path, e) + "/"
 980                                 } else {
 981                                         e = filepath.Join(u.Path, e)
 982                                 }
 983                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
 984                         }
 985                 }
 986         }
 987 }
 988
 989 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 990         arv := arvados.NewClientFromEnv()
 991         var newCollection arvados.Collection
 992         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 993                 "collection": map[string]string{
 994                         "owner_uuid":    arvadostest.ActiveUserUUID,
 995                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
 996                         "name":          "keep-web test collection",
 997                 },
 998                 "ensure_unique_name": true,
 999         })
1000         c.Assert(err, check.IsNil)
1001         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1002
1003         var updated arvados.Collection
1004         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1005                 s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1006                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1007                 req := &http.Request{
1008                         Method:     "DELETE",
1009                         Host:       u.Host,
1010                         URL:        u,
1011                         RequestURI: u.RequestURI(),
1012                         Header: http.Header{
1013                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1014                         },
1015                 }
1016                 resp := httptest.NewRecorder()
1017                 s.testServer.Handler.ServeHTTP(resp, req)
1018                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1019
1020                 updated = arvados.Collection{}
1021                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1022                 c.Check(err, check.IsNil)
1023                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1024                 c.Logf("updated manifest_text %q", updated.ManifestText)
1025         }
1026         c.Check(updated.ManifestText, check.Equals, "")
1027 }
1028
1029 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
1030         s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
1031         authHeader := http.Header{
1032                 "Authorization": {"Bearer " + arvadostest.ManagementToken},
1033         }
1034
1035         resp := httptest.NewRecorder()
1036         u := mustParseURL("http://download.example.com/_health/ping")
1037         req := &http.Request{
1038                 Method:     "GET",
1039                 Host:       u.Host,
1040                 URL:        u,
1041                 RequestURI: u.RequestURI(),
1042                 Header:     authHeader,
1043         }
1044         s.testServer.Handler.ServeHTTP(resp, req)
1045
1046         c.Check(resp.Code, check.Equals, http.StatusOK)
1047         c.Check(resp.Body.String(), check.Matches, `{"health":"OK"}\n`)
1048 }
1049
1050 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1051         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1052
1053         client := s.testServer.Config.Client
1054         client.AuthToken = arvadostest.ActiveToken
1055         arv, err := arvadosclient.New(&client)
1056         c.Assert(err, check.Equals, nil)
1057         kc, err := keepclient.MakeKeepClient(arv)
1058         c.Assert(err, check.Equals, nil)
1059
1060         fs, err := (&arvados.Collection{}).FileSystem(&client, kc)
1061         c.Assert(err, check.IsNil)
1062
1063         trials := []struct {
1064                 filename    string
1065                 content     string
1066                 contentType string
1067         }{
1068                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1069                 {"picture.bmp", "BMX bikes are small this year\n", "image/x-ms-bmp"},
1070                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1071                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1072                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1073         }
1074         for _, trial := range trials {
1075                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1076                 c.Assert(err, check.IsNil)
1077                 _, err = f.Write([]byte(trial.content))
1078                 c.Assert(err, check.IsNil)
1079                 c.Assert(f.Close(), check.IsNil)
1080         }
1081         mtxt, err := fs.MarshalManifest(".")
1082         c.Assert(err, check.IsNil)
1083         var coll arvados.Collection
1084         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1085                 "collection": map[string]string{
1086                         "manifest_text": mtxt,
1087                 },
1088         })
1089         c.Assert(err, check.IsNil)
1090
1091         for _, trial := range trials {
1092                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1093                 req := &http.Request{
1094                         Method:     "GET",
1095                         Host:       u.Host,
1096                         URL:        u,
1097                         RequestURI: u.RequestURI(),
1098                         Header: http.Header{
1099                                 "Authorization": {"Bearer " + client.AuthToken},
1100                         },
1101                 }
1102                 resp := httptest.NewRecorder()
1103                 s.testServer.Handler.ServeHTTP(resp, req)
1104                 c.Check(resp.Code, check.Equals, http.StatusOK)
1105                 c.Check(resp.Header().Get("Content-Type"), check.Equals, trial.contentType)
1106                 c.Check(resp.Body.String(), check.Equals, trial.content)
1107         }
1108 }
1109
1110 func (s *IntegrationSuite) TestKeepClientBlockCache(c *check.C) {
1111         s.testServer.Config.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
1112         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), 42)
1113         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
1114         req := &http.Request{
1115                 Method:     "GET",
1116                 Host:       u.Host,
1117                 URL:        u,
1118                 RequestURI: u.RequestURI(),
1119         }
1120         resp := httptest.NewRecorder()
1121         s.testServer.Handler.ServeHTTP(resp, req)
1122         c.Check(resp.Code, check.Equals, http.StatusOK)
1123         c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, 42)
1124 }
1125
1126 // Writing to a collection shouldn't affect its entry in the
1127 // PDH-to-manifest cache.
1128 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1129         arv, err := arvadosclient.MakeArvadosClient()
1130         c.Assert(err, check.Equals, nil)
1131         arv.ApiToken = arvadostest.ActiveToken
1132
1133         u := mustParseURL("http://x.example/testfile")
1134         req := &http.Request{
1135                 Method:     "GET",
1136                 Host:       u.Host,
1137                 URL:        u,
1138                 RequestURI: u.RequestURI(),
1139                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1140         }
1141
1142         checkWithID := func(id string, status int) {
1143                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1144                 req.Host = req.URL.Host
1145                 resp := httptest.NewRecorder()
1146                 s.testServer.Handler.ServeHTTP(resp, req)
1147                 c.Check(resp.Code, check.Equals, status)
1148         }
1149
1150         var colls [2]arvados.Collection
1151         for i := range colls {
1152                 err := arv.Create("collections",
1153                         map[string]interface{}{
1154                                 "ensure_unique_name": true,
1155                                 "collection": map[string]interface{}{
1156                                         "name": "test collection",
1157                                 },
1158                         }, &colls[i])
1159                 c.Assert(err, check.Equals, nil)
1160         }
1161
1162         // Populate cache with empty collection
1163         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1164
1165         // write a file to colls[0]
1166         reqPut := *req
1167         reqPut.Method = "PUT"
1168         reqPut.URL.Host = colls[0].UUID + ".example"
1169         reqPut.Host = req.URL.Host
1170         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1171         resp := httptest.NewRecorder()
1172         s.testServer.Handler.ServeHTTP(resp, &reqPut)
1173         c.Check(resp.Code, check.Equals, http.StatusCreated)
1174
1175         // new file should not appear in colls[1]
1176         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1177         checkWithID(colls[1].UUID, http.StatusNotFound)
1178
1179         checkWithID(colls[0].UUID, http.StatusOK)
1180 }
1181
1182 func copyHeader(h http.Header) http.Header {
1183         hc := http.Header{}
1184         for k, v := range h {
1185                 hc[k] = append([]string(nil), v...)
1186         }
1187         return hc
1188 }
1189
1190 func (s *IntegrationSuite) TestDownloadLogging(c *check.C) {
1191         h := handler{Config: newConfig(s.ArvConfig)}
1192         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1193         req := &http.Request{
1194                 Method:     "GET",
1195                 Host:       u.Host,
1196                 URL:        u,
1197                 RequestURI: u.RequestURI(),
1198                 Header: http.Header{
1199                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1200                 },
1201         }
1202
1203         var logbuf bytes.Buffer
1204         logger := logrus.New()
1205         logger.Out = &logbuf
1206         resp := httptest.NewRecorder()
1207         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1208         h.ServeHTTP(resp, req)
1209
1210         c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File download".*`)
1211         c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1212 }
1213
1214 func (s *IntegrationSuite) TestUploadLogging(c *check.C) {
1215         defer func() {
1216                 client := s.testServer.Config.Client
1217                 client.RequestAndDecode(nil, "POST", "database/reset", nil, nil)
1218         }()
1219
1220         h := handler{Config: newConfig(s.ArvConfig)}
1221         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/bar")
1222         req := &http.Request{
1223                 Method:     "PUT",
1224                 Host:       u.Host,
1225                 URL:        u,
1226                 RequestURI: u.RequestURI(),
1227                 Header: http.Header{
1228                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1229                 },
1230                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1231         }
1232
1233         var logbuf bytes.Buffer
1234         logger := logrus.New()
1235         logger.Out = &logbuf
1236         resp := httptest.NewRecorder()
1237         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1238         h.ServeHTTP(resp, req)
1239
1240         c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File upload".*`)
1241         c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1242 }
1243
1244 func (s *IntegrationSuite) TestDownloadPermission(c *check.C) {
1245         config := newConfig(s.ArvConfig)
1246         h := handler{Config: config}
1247         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1248
1249         for _, adminperm := range []bool{true, false} {
1250                 for _, userperm := range []bool{true, false} {
1251
1252                         config.cluster.Collections.KeepWebPermission.Admin.Download = adminperm
1253                         config.cluster.Collections.KeepWebPermission.User.Download = userperm
1254
1255                         // Test admin permission
1256                         req := &http.Request{
1257                                 Method:     "GET",
1258                                 Host:       u.Host,
1259                                 URL:        u,
1260                                 RequestURI: u.RequestURI(),
1261                                 Header: http.Header{
1262                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1263                                 },
1264                         }
1265
1266                         var logbuf bytes.Buffer
1267                         logger := logrus.New()
1268                         logger.Out = &logbuf
1269                         resp := httptest.NewRecorder()
1270                         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1271                         h.ServeHTTP(resp, req)
1272
1273                         if adminperm {
1274                                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusOK)
1275                                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File download".*`)
1276                                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1277                         } else {
1278                                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1279                                 c.Check(logbuf.String(), check.Equals, "")
1280                         }
1281
1282                         // Test user permission
1283                         req = &http.Request{
1284                                 Method:     "GET",
1285                                 Host:       u.Host,
1286                                 URL:        u,
1287                                 RequestURI: u.RequestURI(),
1288                                 Header: http.Header{
1289                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1290                                 },
1291                         }
1292
1293                         logbuf = bytes.Buffer{}
1294                         logger = logrus.New()
1295                         logger.Out = &logbuf
1296                         resp = httptest.NewRecorder()
1297                         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1298                         h.ServeHTTP(resp, req)
1299
1300                         if userperm {
1301                                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusOK)
1302                                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File download".*`)
1303                                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1304                         } else {
1305                                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1306                                 c.Check(logbuf.String(), check.Equals, "")
1307                         }
1308                 }
1309         }
1310 }
1311
1312 func (s *IntegrationSuite) TestUploadPermission(c *check.C) {
1313         defer func() {
1314                 client := s.testServer.Config.Client
1315                 client.RequestAndDecode(nil, "POST", "database/reset", nil, nil)
1316         }()
1317
1318         config := newConfig(s.ArvConfig)
1319         h := handler{Config: config}
1320         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
1321
1322         for _, adminperm := range []bool{true, false} {
1323                 for _, userperm := range []bool{true, false} {
1324
1325                         config.cluster.Collections.KeepWebPermission.Admin.Upload = adminperm
1326                         config.cluster.Collections.KeepWebPermission.User.Upload = userperm
1327
1328                         // Test admin permission
1329                         req := &http.Request{
1330                                 Method:     "PUT",
1331                                 Host:       u.Host,
1332                                 URL:        u,
1333                                 RequestURI: u.RequestURI(),
1334                                 Header: http.Header{
1335                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
1336                                 },
1337                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1338                         }
1339
1340                         var logbuf bytes.Buffer
1341                         logger := logrus.New()
1342                         logger.Out = &logbuf
1343                         resp := httptest.NewRecorder()
1344                         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1345                         h.ServeHTTP(resp, req)
1346
1347                         if adminperm {
1348                                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusCreated)
1349                                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File upload".*`)
1350                                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1351                         } else {
1352                                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1353                                 c.Check(logbuf.String(), check.Equals, "")
1354                         }
1355
1356                         // Test user permission
1357                         req = &http.Request{
1358                                 Method:     "PUT",
1359                                 Host:       u.Host,
1360                                 URL:        u,
1361                                 RequestURI: u.RequestURI(),
1362                                 Header: http.Header{
1363                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
1364                                 },
1365                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
1366                         }
1367
1368                         logbuf = bytes.Buffer{}
1369                         logger = logrus.New()
1370                         logger.Out = &logbuf
1371                         resp = httptest.NewRecorder()
1372                         req = req.WithContext(ctxlog.Context(context.Background(), logger))
1373                         h.ServeHTTP(resp, req)
1374
1375                         if userperm {
1376                                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusCreated)
1377                                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File upload".*`)
1378                                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
1379                         } else {
1380                                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
1381                                 c.Check(logbuf.String(), check.Equals, "")
1382                         }
1383                 }
1384         }
1385 }