services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "sync"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         "golang.org/x/net/html"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(&bytes.Buffer{}, logger)
  50         cfg, err := ldr.Load()
  51         c.Assert(err, check.IsNil)
  52         cc, err := cfg.GetCluster("")
  53         c.Assert(err, check.IsNil)
  54         s.cluster = cc
  55         s.handler = &handler{
  56                 Cluster: cc,
  57                 Cache: cache{
  58                         cluster:  cc,
  59                         logger:   logger,
  60                         registry: prometheus.NewRegistry(),
  61                 },
  62                 metrics: newMetrics(prometheus.NewRegistry()),
  63         }
  64 }
  65
  66 func newCollection(collID string) *arvados.Collection {
  67         coll := &arvados.Collection{UUID: collID}
  68         manifestKey := collID
  69         if pdh, ok := arvadostest.TestCollectionUUIDToPDH[collID]; ok {
  70                 coll.PortableDataHash = pdh
  71                 manifestKey = pdh
  72         }
  73         if mtext, ok := arvadostest.TestCollectionPDHToManifest[manifestKey]; ok {
  74                 coll.ManifestText = mtext
  75         }
  76         return coll
  77 }
  78
  79 func newRequest(method, urlStr string) *http.Request {
  80         u := mustParseURL(urlStr)
  81         return &http.Request{
  82                 Method:     method,
  83                 Host:       u.Host,
  84                 URL:        u,
  85                 RequestURI: u.RequestURI(),
  86                 RemoteAddr: "10.20.30.40:56789",
  87                 Header:     http.Header{},
  88         }
  89 }
  90
  91 func newLoggerAndContext() (*bytes.Buffer, context.Context) {
  92         var logbuf bytes.Buffer
  93         logger := logrus.New()
  94         logger.Out = &logbuf
  95         return &logbuf, ctxlog.Context(context.Background(), logger)
  96 }
  97
  98 func (s *UnitSuite) TestLogEventTypes(c *check.C) {
  99         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 100         for method, expected := range map[string]string{
 101                 "GET":  "file_download",
 102                 "POST": "file_upload",
 103                 "PUT":  "file_upload",
 104         } {
 105                 filePath := "/" + method
 106                 req := newRequest(method, collURL+filePath)
 107                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 108                 if !c.Check(actual, check.NotNil) {
 109                         continue
 110                 }
 111                 c.Check(actual.eventType, check.Equals, expected)
 112         }
 113 }
 114
 115 func (s *UnitSuite) TestUnloggedEventTypes(c *check.C) {
 116         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 117         for _, method := range []string{"DELETE", "HEAD", "OPTIONS", "PATCH"} {
 118                 filePath := "/" + method
 119                 req := newRequest(method, collURL+filePath)
 120                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 121                 c.Check(actual, check.IsNil,
 122                         check.Commentf("%s request made a log event", method))
 123         }
 124 }
 125
 126 func (s *UnitSuite) TestLogFilePath(c *check.C) {
 127         coll := newCollection(arvadostest.FooCollection)
 128         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 129         for _, filePath := range []string{"/foo", "/Foo", "/foo/bar"} {
 130                 req := newRequest("GET", collURL+filePath)
 131                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 132                 if !c.Check(actual, check.NotNil) {
 133                         continue
 134                 }
 135                 c.Check(actual.collFilePath, check.Equals, filePath)
 136         }
 137 }
 138
 139 func (s *UnitSuite) TestLogRemoteAddr(c *check.C) {
 140         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 141         filePath := "/foo"
 142         req := newRequest("GET", collURL+filePath)
 143
 144         for _, addr := range []string{"10.20.30.55", "192.168.144.120", "192.0.2.4"} {
 145                 req.RemoteAddr = addr + ":57914"
 146                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 147                 if !c.Check(actual, check.NotNil) {
 148                         continue
 149                 }
 150                 c.Check(actual.clientAddr, check.Equals, addr)
 151         }
 152
 153         for _, addr := range []string{"100::20:30:40", "2001:db8::90:100", "3fff::30"} {
 154                 req.RemoteAddr = fmt.Sprintf("[%s]:57916", addr)
 155                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 156                 if !c.Check(actual, check.NotNil) {
 157                         continue
 158                 }
 159                 c.Check(actual.clientAddr, check.Equals, addr)
 160         }
 161 }
 162
 163 func (s *UnitSuite) TestLogXForwardedFor(c *check.C) {
 164         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 165         filePath := "/foo"
 166         req := newRequest("GET", collURL+filePath)
 167         for xff, expected := range map[string]string{
 168                 "10.20.30.55":                          "10.20.30.55",
 169                 "192.168.144.120, 10.20.30.120":        "10.20.30.120",
 170                 "192.0.2.4, 192.0.2.6, 192.0.2.8":      "192.0.2.8",
 171                 "192.0.2.4,192.168.2.4":                "192.168.2.4",
 172                 "10.20.30.60,192.168.144.40,192.0.2.4": "192.0.2.4",
 173                 "100::20:30:50":                        "100::20:30:50",
 174                 "2001:db8::80:90, 100::100":            "100::100",
 175                 "3fff::ff, 3fff::ee, 3fff::fe":         "3fff::fe",
 176                 "3fff::3f,100::1000":                   "100::1000",
 177                 "2001:db8::88,100::88,3fff::88":        "3fff::88",
 178                 "10.20.30.60, 2001:db8::60":            "2001:db8::60",
 179                 "2001:db8::20,10.20.30.20":             "10.20.30.20",
 180                 ", 10.20.30.123, 100::123":             "100::123",
 181                 ",100::321,10.30.20.10":                "10.30.20.10",
 182         } {
 183                 req.Header.Set("X-Forwarded-For", xff)
 184                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 185                 if !c.Check(actual, check.NotNil) {
 186                         continue
 187                 }
 188                 c.Check(actual.clientAddr, check.Equals, expected)
 189         }
 190 }
 191
 192 func (s *UnitSuite) TestLogXForwardedForMalformed(c *check.C) {
 193         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 194         filePath := "/foo"
 195         req := newRequest("GET", collURL+filePath)
 196         for _, xff := range []string{"", ",", "10.20,30.40", "foo, bar"} {
 197                 req.Header.Set("X-Forwarded-For", xff)
 198                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 199                 if !c.Check(actual, check.NotNil) {
 200                         continue
 201                 }
 202                 c.Check(actual.clientAddr, check.Equals, "10.20.30.40")
 203         }
 204 }
 205
 206 func (s *UnitSuite) TestLogXForwardedForMultivalue(c *check.C) {
 207         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 208         filePath := "/foo"
 209         req := newRequest("GET", collURL+filePath)
 210         req.Header.Set("X-Forwarded-For", ", ")
 211         req.Header.Add("X-Forwarded-For", "2001:db8::db9:dbd")
 212         req.Header.Add("X-Forwarded-For", "10.20.30.90")
 213         actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 214         c.Assert(actual, check.NotNil)
 215         c.Check(actual.clientAddr, check.Equals, "10.20.30.90")
 216 }
 217
 218 func (s *UnitSuite) TestLogClientAddressCanonicalization(c *check.C) {
 219         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 220         filePath := "/foo"
 221         req := newRequest("GET", collURL+filePath)
 222         expected := "2001:db8::12:0"
 223
 224         req.RemoteAddr = "[2001:db8::012:0000]:57918"
 225         a := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 226         c.Assert(a, check.NotNil)
 227         c.Check(a.clientAddr, check.Equals, expected)
 228
 229         req.RemoteAddr = "10.20.30.40:57919"
 230         req.Header.Set("X-Forwarded-For", "2001:db8:0::0:12:00")
 231         b := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 232         c.Assert(b, check.NotNil)
 233         c.Check(b.clientAddr, check.Equals, expected)
 234 }
 235
 236 func (s *UnitSuite) TestLogAnonymousUser(c *check.C) {
 237         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 238         filePath := "/foo"
 239         req := newRequest("GET", collURL+filePath)
 240         actual := newFileEventLog(s.handler, req, filePath, nil, nil, arvadostest.AnonymousToken)
 241         c.Assert(actual, check.NotNil)
 242         c.Check(actual.userUUID, check.Equals, s.handler.Cluster.ClusterID+"-tpzed-anonymouspublic")
 243         c.Check(actual.userFullName, check.Equals, "")
 244         c.Check(actual.clientToken, check.Equals, arvadostest.AnonymousToken)
 245 }
 246
 247 func (s *UnitSuite) TestLogUser(c *check.C) {
 248         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 249         for _, trial := range []struct{ uuid, fullName, token string }{
 250                 {arvadostest.ActiveUserUUID, "Active User", arvadostest.ActiveToken},
 251                 {arvadostest.SpectatorUserUUID, "Spectator User", arvadostest.SpectatorToken},
 252         } {
 253                 filePath := "/" + trial.uuid
 254                 req := newRequest("GET", collURL+filePath)
 255                 user := &arvados.User{
 256                         UUID:     trial.uuid,
 257                         FullName: trial.fullName,
 258                 }
 259                 actual := newFileEventLog(s.handler, req, filePath, nil, user, trial.token)
 260                 if !c.Check(actual, check.NotNil) {
 261                         continue
 262                 }
 263                 c.Check(actual.userUUID, check.Equals, trial.uuid)
 264                 c.Check(actual.userFullName, check.Equals, trial.fullName)
 265                 c.Check(actual.clientToken, check.Equals, trial.token)
 266         }
 267 }
 268
 269 func (s *UnitSuite) TestLogCollectionByUUID(c *check.C) {
 270         for collUUID, collPDH := range arvadostest.TestCollectionUUIDToPDH {
 271                 collURL := "http://keep-web.example/c=" + collUUID
 272                 filePath := "/" + collUUID
 273                 req := newRequest("GET", collURL+filePath)
 274                 coll := newCollection(collUUID)
 275                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 276                 if !c.Check(actual, check.NotNil) {
 277                         continue
 278                 }
 279                 c.Check(actual.collUUID, check.Equals, collUUID)
 280                 c.Check(actual.collPDH, check.Equals, collPDH)
 281         }
 282 }
 283
 284 func (s *UnitSuite) TestLogCollectionByPDH(c *check.C) {
 285         for _, collPDH := range arvadostest.TestCollectionUUIDToPDH {
 286                 collURL := "http://keep-web.example/c=" + collPDH
 287                 filePath := "/PDHFile"
 288                 req := newRequest("GET", collURL+filePath)
 289                 coll := newCollection(collPDH)
 290                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 291                 if !c.Check(actual, check.NotNil) {
 292                         continue
 293                 }
 294                 c.Check(actual.collPDH, check.Equals, collPDH)
 295                 c.Check(actual.collUUID, check.Equals, "")
 296         }
 297 }
 298
 299 func (s *UnitSuite) TestLogGETUUIDAsDict(c *check.C) {
 300         filePath := "/foo"
 301         reqPath := "/c=" + arvadostest.FooCollection + filePath
 302         req := newRequest("GET", "http://keep-web.example"+reqPath)
 303         coll := newCollection(arvadostest.FooCollection)
 304         logEvent := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 305         c.Assert(logEvent, check.NotNil)
 306         c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 307                 "event_type":  "file_download",
 308                 "object_uuid": s.handler.Cluster.ClusterID + "-tpzed-anonymouspublic",
 309                 "properties": arvadosclient.Dict{
 310                         "reqPath":              reqPath,
 311                         "collection_uuid":      arvadostest.FooCollection,
 312                         "collection_file_path": filePath,
 313                         "portable_data_hash":   arvadostest.FooCollectionPDH,
 314                 },
 315         })
 316 }
 317
 318 func (s *UnitSuite) TestLogGETPDHAsDict(c *check.C) {
 319         filePath := "/Foo"
 320         reqPath := "/c=" + arvadostest.FooCollectionPDH + filePath
 321         req := newRequest("GET", "http://keep-web.example"+reqPath)
 322         coll := newCollection(arvadostest.FooCollectionPDH)
 323         user := &arvados.User{
 324                 UUID:     arvadostest.ActiveUserUUID,
 325                 FullName: "Active User",
 326         }
 327         logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 328         c.Assert(logEvent, check.NotNil)
 329         c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 330                 "event_type":  "file_download",
 331                 "object_uuid": arvadostest.ActiveUserUUID,
 332                 "properties": arvadosclient.Dict{
 333                         "reqPath":              reqPath,
 334                         "portable_data_hash":   arvadostest.FooCollectionPDH,
 335                         "collection_uuid":      "",
 336                         "collection_file_path": filePath,
 337                 },
 338         })
 339 }
 340
 341 func (s *UnitSuite) TestLogUploadAsDict(c *check.C) {
 342         coll := newCollection(arvadostest.FooCollection)
 343         user := &arvados.User{
 344                 UUID:     arvadostest.ActiveUserUUID,
 345                 FullName: "Active User",
 346         }
 347         for _, method := range []string{"POST", "PUT"} {
 348                 filePath := "/" + method + "File"
 349                 reqPath := "/c=" + arvadostest.FooCollection + filePath
 350                 req := newRequest(method, "http://keep-web.example"+reqPath)
 351                 logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 352                 if !c.Check(logEvent, check.NotNil) {
 353                         continue
 354                 }
 355                 c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 356                         "event_type":  "file_upload",
 357                         "object_uuid": arvadostest.ActiveUserUUID,
 358                         "properties": arvadosclient.Dict{
 359                                 "reqPath":              reqPath,
 360                                 "collection_uuid":      arvadostest.FooCollection,
 361                                 "collection_file_path": filePath,
 362                         },
 363                 })
 364         }
 365 }
 366
 367 func (s *UnitSuite) TestLogGETUUIDAsFields(c *check.C) {
 368         filePath := "/foo"
 369         reqPath := "/c=" + arvadostest.FooCollection + filePath
 370         req := newRequest("GET", "http://keep-web.example"+reqPath)
 371         coll := newCollection(arvadostest.FooCollection)
 372         logEvent := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 373         c.Assert(logEvent, check.NotNil)
 374         c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 375                 "user_uuid":            s.handler.Cluster.ClusterID + "-tpzed-anonymouspublic",
 376                 "collection_uuid":      arvadostest.FooCollection,
 377                 "collection_file_path": filePath,
 378                 "portable_data_hash":   arvadostest.FooCollectionPDH,
 379         })
 380 }
 381
 382 func (s *UnitSuite) TestLogGETPDHAsFields(c *check.C) {
 383         filePath := "/Foo"
 384         reqPath := "/c=" + arvadostest.FooCollectionPDH + filePath
 385         req := newRequest("GET", "http://keep-web.example"+reqPath)
 386         coll := newCollection(arvadostest.FooCollectionPDH)
 387         user := &arvados.User{
 388                 UUID:     arvadostest.ActiveUserUUID,
 389                 FullName: "Active User",
 390         }
 391         logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 392         c.Assert(logEvent, check.NotNil)
 393         c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 394                 "user_uuid":            arvadostest.ActiveUserUUID,
 395                 "user_full_name":       "Active User",
 396                 "collection_uuid":      "",
 397                 "collection_file_path": filePath,
 398                 "portable_data_hash":   arvadostest.FooCollectionPDH,
 399         })
 400 }
 401
 402 func (s *UnitSuite) TestLogUploadAsFields(c *check.C) {
 403         coll := newCollection(arvadostest.FooCollection)
 404         user := &arvados.User{
 405                 UUID:     arvadostest.ActiveUserUUID,
 406                 FullName: "Active User",
 407         }
 408         for _, method := range []string{"POST", "PUT"} {
 409                 filePath := "/" + method + "File"
 410                 reqPath := "/c=" + arvadostest.FooCollection + filePath
 411                 req := newRequest(method, "http://keep-web.example"+reqPath)
 412                 logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 413                 if !c.Check(logEvent, check.NotNil) {
 414                         continue
 415                 }
 416                 c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 417                         "user_uuid":            arvadostest.ActiveUserUUID,
 418                         "user_full_name":       "Active User",
 419                         "collection_uuid":      arvadostest.FooCollection,
 420                         "collection_file_path": filePath,
 421                 })
 422         }
 423 }
 424
 425 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
 426         h := s.handler
 427         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
 428         req := &http.Request{
 429                 Method:     "OPTIONS",
 430                 Host:       u.Host,
 431                 URL:        u,
 432                 RequestURI: u.RequestURI(),
 433                 Header: http.Header{
 434                         "Origin":                        {"https://workbench.example"},
 435                         "Access-Control-Request-Method": {"POST"},
 436                 },
 437         }
 438
 439         // Check preflight for an allowed request
 440         resp := httptest.NewRecorder()
 441         h.ServeHTTP(resp, req)
 442         c.Check(resp.Code, check.Equals, http.StatusOK)
 443         c.Check(resp.Body.String(), check.Equals, "")
 444         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 445         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
 446         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
 447
 448         // Check preflight for a disallowed request
 449         resp = httptest.NewRecorder()
 450         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
 451         h.ServeHTTP(resp, req)
 452         c.Check(resp.Body.String(), check.Equals, "")
 453         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
 454 }
 455
 456 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
 457         for _, trial := range []struct {
 458                 method   string
 459                 path     string
 460                 prefix   string
 461                 source   string
 462                 notFound bool
 463                 seeOther bool
 464         }{
 465                 {
 466                         method: "PROPFIND",
 467                         path:   "/",
 468                 },
 469                 {
 470                         method: "PROPFIND",
 471                         path:   "/dir1",
 472                 },
 473                 {
 474                         method: "PROPFIND",
 475                         path:   "/dir1/",
 476                 },
 477                 {
 478                         method: "PROPFIND",
 479                         path:   "/dir1/foo",
 480                         prefix: "/dir1",
 481                         source: "/dir1",
 482                 },
 483                 {
 484                         method: "PROPFIND",
 485                         path:   "/prefix/dir1/foo",
 486                         prefix: "/prefix/",
 487                         source: "",
 488                 },
 489                 {
 490                         method: "PROPFIND",
 491                         path:   "/prefix/dir1/foo",
 492                         prefix: "/prefix",
 493                         source: "",
 494                 },
 495                 {
 496                         method: "PROPFIND",
 497                         path:   "/prefix/dir1/foo",
 498                         prefix: "/prefix/",
 499                         source: "/",
 500                 },
 501                 {
 502                         method: "PROPFIND",
 503                         path:   "/prefix/foo",
 504                         prefix: "/prefix/",
 505                         source: "/dir1/",
 506                 },
 507                 {
 508                         method: "GET",
 509                         path:   "/prefix/foo",
 510                         prefix: "/prefix/",
 511                         source: "/dir1/",
 512                 },
 513                 {
 514                         method: "PROPFIND",
 515                         path:   "/prefix/",
 516                         prefix: "/prefix",
 517                         source: "/dir1",
 518                 },
 519                 {
 520                         method: "PROPFIND",
 521                         path:   "/prefix",
 522                         prefix: "/prefix",
 523                         source: "/dir1/",
 524                 },
 525                 {
 526                         method:   "GET",
 527                         path:     "/prefix",
 528                         prefix:   "/prefix",
 529                         source:   "/dir1",
 530                         seeOther: true,
 531                 },
 532                 {
 533                         method:   "PROPFIND",
 534                         path:     "/dir1/foo",
 535                         prefix:   "",
 536                         source:   "/dir1",
 537                         notFound: true,
 538                 },
 539         } {
 540                 c.Logf("trial %+v", trial)
 541                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 542                 req := &http.Request{
 543                         Method:     trial.method,
 544                         Host:       u.Host,
 545                         URL:        u,
 546                         RequestURI: u.RequestURI(),
 547                         Header: http.Header{
 548                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 549                                 "X-Webdav-Prefix": {trial.prefix},
 550                                 "X-Webdav-Source": {trial.source},
 551                         },
 552                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 553                 }
 554
 555                 resp := httptest.NewRecorder()
 556                 s.handler.ServeHTTP(resp, req)
 557                 if trial.notFound {
 558                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 559                 } else if trial.method == "PROPFIND" {
 560                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 561                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 562                 } else if trial.seeOther {
 563                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 564                 } else {
 565                         c.Check(resp.Code, check.Equals, http.StatusOK)
 566                 }
 567         }
 568 }
 569
 570 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 571         // Ensure we start with an empty cache
 572         defer os.Setenv("HOME", os.Getenv("HOME"))
 573         os.Setenv("HOME", c.MkDir())
 574         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(0)
 575
 576         for _, trial := range []struct {
 577                 dataExists    bool
 578                 sendIMSHeader bool
 579                 expectStatus  int
 580                 logRegexp     string
 581         }{
 582                 // If we return no content due to a Keep read error,
 583                 // we should emit a log message.
 584                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 585
 586                 // If we return no content because the client sent an
 587                 // If-Modified-Since header, our response should be
 588                 // 304.  We still expect a "File download" log since it
 589                 // counts as a file access for auditing.
 590                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 591         } {
 592                 c.Logf("trial: %+v", trial)
 593                 arvadostest.StartKeep(2, true)
 594                 if trial.dataExists {
 595                         arv, err := arvadosclient.MakeArvadosClient()
 596                         c.Assert(err, check.IsNil)
 597                         arv.ApiToken = arvadostest.ActiveToken
 598                         kc, err := keepclient.MakeKeepClient(arv)
 599                         c.Assert(err, check.IsNil)
 600                         _, _, err = kc.PutB([]byte("foo"))
 601                         c.Assert(err, check.IsNil)
 602                 }
 603
 604                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 605                 req := &http.Request{
 606                         Method:     "GET",
 607                         Host:       u.Host,
 608                         URL:        u,
 609                         RequestURI: u.RequestURI(),
 610                         Header: http.Header{
 611                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 612                         },
 613                 }
 614                 if trial.sendIMSHeader {
 615                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 616                 }
 617
 618                 var logbuf bytes.Buffer
 619                 logger := logrus.New()
 620                 logger.Out = &logbuf
 621                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 622
 623                 resp := httptest.NewRecorder()
 624                 s.handler.ServeHTTP(resp, req)
 625                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 626                 c.Check(resp.Body.String(), check.Equals, "")
 627
 628                 c.Log(logbuf.String())
 629                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 630         }
 631 }
 632
 633 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 634         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 635         token := arvadostest.ActiveToken
 636         for _, trial := range []string{
 637                 "http://keep-web/c=" + bogusID + "/foo",
 638                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 639                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 640                 "http://keep-web/collections/" + bogusID + "/foo",
 641                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 642                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 643         } {
 644                 c.Log(trial)
 645                 u := mustParseURL(trial)
 646                 req := &http.Request{
 647                         Method:     "GET",
 648                         Host:       u.Host,
 649                         URL:        u,
 650                         RequestURI: u.RequestURI(),
 651                 }
 652                 resp := httptest.NewRecorder()
 653                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 654                 s.handler.ServeHTTP(resp, req)
 655                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 656         }
 657 }
 658
 659 func mustParseURL(s string) *url.URL {
 660         r, err := url.Parse(s)
 661         if err != nil {
 662                 panic("parse URL: " + s)
 663         }
 664         return r
 665 }
 666
 667 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 668         for _, testURL := range []string{
 669                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 670                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 671         } {
 672                 resp := httptest.NewRecorder()
 673                 u := mustParseURL(testURL)
 674                 req := &http.Request{
 675                         Method:     "GET",
 676                         URL:        u,
 677                         RequestURI: u.RequestURI(),
 678                 }
 679                 s.handler.ServeHTTP(resp, req)
 680                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 681                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 682         }
 683 }
 684
 685 // An authorizer modifies an HTTP request to make use of the given
 686 // token -- by adding it to a header, cookie, query param, or whatever
 687 // -- and returns the HTTP status code we should expect from keep-web if
 688 // the token is invalid.
 689 type authorizer func(*http.Request, string) int
 690
 691 // We still need to accept "OAuth2 ..." as equivalent to "Bearer ..."
 692 // for compatibility with older clients, including SDKs before 3.0.
 693 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 694         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 695 }
 696 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 697         r.Header.Add("Authorization", "OAuth2 "+tok)
 698         return http.StatusUnauthorized
 699 }
 700
 701 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 702         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 703 }
 704 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 705         r.Header.Add("Authorization", "Bearer "+tok)
 706         return http.StatusUnauthorized
 707 }
 708
 709 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 710         s.doVhostRequests(c, authzViaCookieValue)
 711 }
 712 func authzViaCookieValue(r *http.Request, tok string) int {
 713         r.AddCookie(&http.Cookie{
 714                 Name:  "arvados_api_token",
 715                 Value: auth.EncodeTokenCookie([]byte(tok)),
 716         })
 717         return http.StatusUnauthorized
 718 }
 719
 720 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 721         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 722 }
 723 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 724         r.AddCookie(&http.Cookie{
 725                 Name:  "arvados_api_token",
 726                 Value: auth.EncodeTokenCookie([]byte(tok)),
 727         })
 728         return http.StatusUnauthorized
 729 }
 730
 731 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 732         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 733                 r.AddCookie(&http.Cookie{
 734                         Name:  "arvados_api_token",
 735                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 736                 })
 737                 return http.StatusUnauthorized
 738         })
 739 }
 740
 741 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 742         s.doVhostRequests(c, authzViaPath)
 743 }
 744 func authzViaPath(r *http.Request, tok string) int {
 745         r.URL.Path = "/t=" + tok + r.URL.Path
 746         return http.StatusNotFound
 747 }
 748
 749 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 750         s.doVhostRequests(c, authzViaQueryString)
 751 }
 752 func authzViaQueryString(r *http.Request, tok string) int {
 753         r.URL.RawQuery = "api_token=" + tok
 754         return http.StatusUnauthorized
 755 }
 756
 757 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 758         s.doVhostRequests(c, authzViaPOST)
 759 }
 760 func authzViaPOST(r *http.Request, tok string) int {
 761         r.Method = "POST"
 762         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 763         r.Body = ioutil.NopCloser(strings.NewReader(
 764                 url.Values{"api_token": {tok}}.Encode()))
 765         return http.StatusUnauthorized
 766 }
 767
 768 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 769         s.doVhostRequests(c, authzViaPOST)
 770 }
 771 func authzViaXHRPOST(r *http.Request, tok string) int {
 772         r.Method = "POST"
 773         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 774         r.Header.Add("Origin", "https://origin.example")
 775         r.Body = ioutil.NopCloser(strings.NewReader(
 776                 url.Values{
 777                         "api_token":   {tok},
 778                         "disposition": {"attachment"},
 779                 }.Encode()))
 780         return http.StatusUnauthorized
 781 }
 782
 783 // Try some combinations of {url, token} using the given authorization
 784 // mechanism, and verify the result is correct.
 785 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 786         for _, hostPath := range []string{
 787                 arvadostest.FooCollection + ".example.com/foo",
 788                 arvadostest.FooCollection + "--collections.example.com/foo",
 789                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 790                 arvadostest.FooCollectionPDH + ".example.com/foo",
 791                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 792                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 793         } {
 794                 c.Log("doRequests: ", hostPath)
 795                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 796         }
 797 }
 798
 799 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 800         for _, tok := range []string{
 801                 arvadostest.ActiveToken,
 802                 arvadostest.ActiveToken[:15],
 803                 arvadostest.SpectatorToken,
 804                 "bogus",
 805                 "",
 806         } {
 807                 u := mustParseURL("http://" + hostPath)
 808                 req := &http.Request{
 809                         Method:     "GET",
 810                         Host:       u.Host,
 811                         URL:        u,
 812                         RequestURI: u.RequestURI(),
 813                         Header:     http.Header{},
 814                 }
 815                 failCode := authz(req, tok)
 816                 req, resp := s.doReq(req)
 817                 code, body := resp.Code, resp.Body.String()
 818
 819                 // If the initial request had a (non-empty) token
 820                 // showing in the query string, we should have been
 821                 // redirected in order to hide it in a cookie.
 822                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 823
 824                 if tok == arvadostest.ActiveToken {
 825                         c.Check(code, check.Equals, http.StatusOK)
 826                         c.Check(body, check.Equals, "foo")
 827                 } else {
 828                         c.Check(code >= 400, check.Equals, true)
 829                         c.Check(code < 500, check.Equals, true)
 830                         if tok == arvadostest.SpectatorToken {
 831                                 // Valid token never offers to retry
 832                                 // with different credentials.
 833                                 c.Check(code, check.Equals, http.StatusNotFound)
 834                         } else {
 835                                 // Invalid token can ask to retry
 836                                 // depending on the authz method.
 837                                 c.Check(code, check.Equals, failCode)
 838                         }
 839                         if code == 404 {
 840                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 841                         } else {
 842                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 843                         }
 844                 }
 845         }
 846 }
 847
 848 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 849         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 850                 for _, port := range []string{"80", "443", "8000"} {
 851                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 852                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 853                         req := &http.Request{
 854                                 Method:     "GET",
 855                                 Host:       u.Host,
 856                                 URL:        u,
 857                                 RequestURI: u.RequestURI(),
 858                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 859                         }
 860                         req, resp := s.doReq(req)
 861                         code, _ := resp.Code, resp.Body.String()
 862
 863                         if port == "8000" {
 864                                 c.Check(code, check.Equals, 401)
 865                         } else {
 866                                 c.Check(code, check.Equals, 200)
 867                         }
 868                 }
 869         }
 870 }
 871
 872 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 873         u := mustParseURL(urlstring)
 874         if hdr == nil && token != "" {
 875                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 876         } else if hdr == nil {
 877                 hdr = http.Header{}
 878         } else if token != "" {
 879                 panic("must not pass both token and hdr")
 880         }
 881         return s.doReq(&http.Request{
 882                 Method:     method,
 883                 Host:       u.Host,
 884                 URL:        u,
 885                 RequestURI: u.RequestURI(),
 886                 Header:     hdr,
 887         })
 888 }
 889
 890 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 891         resp := httptest.NewRecorder()
 892         s.handler.ServeHTTP(resp, req)
 893         if resp.Code != http.StatusSeeOther {
 894                 return req, resp
 895         }
 896         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 897         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 898         req = &http.Request{
 899                 Method:     "GET",
 900                 Host:       u.Host,
 901                 URL:        u,
 902                 RequestURI: u.RequestURI(),
 903                 Header:     http.Header{},
 904         }
 905         for _, c := range cookies {
 906                 req.AddCookie(c)
 907         }
 908         return s.doReq(req)
 909 }
 910
 911 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 912         s.testVhostRedirectTokenToCookie(c, "GET",
 913                 arvadostest.FooCollection+".example.com/foo",
 914                 "?api_token="+arvadostest.ActiveToken,
 915                 nil,
 916                 "",
 917                 http.StatusOK,
 918                 "foo",
 919         )
 920 }
 921
 922 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 923         s.testVhostRedirectTokenToCookie(c, "GET",
 924                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 925                 "",
 926                 nil,
 927                 "",
 928                 http.StatusOK,
 929                 "foo",
 930         )
 931 }
 932
 933 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 934         s.testVhostRedirectTokenToCookie(c, "GET",
 935                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 936                 "",
 937                 nil,
 938                 "",
 939                 http.StatusOK,
 940                 "foo",
 941         )
 942         // Same valid sharing token, but requesting a different collection
 943         s.testVhostRedirectTokenToCookie(c, "GET",
 944                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 945                 "",
 946                 nil,
 947                 "",
 948                 http.StatusNotFound,
 949                 regexp.QuoteMeta(notFoundMessage+"\n"),
 950         )
 951 }
 952
 953 // Bad token in URL is 404 Not Found because it doesn't make sense to
 954 // retry the same URL with different authorization.
 955 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 956         s.testVhostRedirectTokenToCookie(c, "GET",
 957                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 958                 "",
 959                 nil,
 960                 "",
 961                 http.StatusNotFound,
 962                 regexp.QuoteMeta(notFoundMessage+"\n"),
 963         )
 964 }
 965
 966 // Bad token in a cookie (even if it got there via our own
 967 // query-string-to-cookie redirect) is, in principle, retryable via
 968 // wb2-login-and-redirect flow.
 969 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 970         // Inline
 971         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 972                 arvadostest.FooCollection+".example.com/foo",
 973                 "?api_token=thisisabogustoken",
 974                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 975                 "",
 976                 http.StatusSeeOther,
 977                 "",
 978         )
 979         u, err := url.Parse(resp.Header().Get("Location"))
 980         c.Assert(err, check.IsNil)
 981         c.Logf("redirected to %s", u)
 982         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 983         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 984         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 985
 986         // Download/attachment indicated by ?disposition=attachment
 987         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 988                 arvadostest.FooCollection+".example.com/foo",
 989                 "?api_token=thisisabogustoken&disposition=attachment",
 990                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 991                 "",
 992                 http.StatusSeeOther,
 993                 "",
 994         )
 995         u, err = url.Parse(resp.Header().Get("Location"))
 996         c.Assert(err, check.IsNil)
 997         c.Logf("redirected to %s", u)
 998         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 999         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
1000         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1001
1002         // Download/attachment indicated by vhost
1003         resp = s.testVhostRedirectTokenToCookie(c, "GET",
1004                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1005                 "?api_token=thisisabogustoken",
1006                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
1007                 "",
1008                 http.StatusSeeOther,
1009                 "",
1010         )
1011         u, err = url.Parse(resp.Header().Get("Location"))
1012         c.Assert(err, check.IsNil)
1013         c.Logf("redirected to %s", u)
1014         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1015         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
1016         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1017
1018         // Without "Sec-Fetch-Mode: navigate" header, just 401.
1019         s.testVhostRedirectTokenToCookie(c, "GET",
1020                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1021                 "?api_token=thisisabogustoken",
1022                 http.Header{"Sec-Fetch-Mode": {"cors"}},
1023                 "",
1024                 http.StatusUnauthorized,
1025                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
1026         )
1027         s.testVhostRedirectTokenToCookie(c, "GET",
1028                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1029                 "?api_token=thisisabogustoken",
1030                 nil,
1031                 "",
1032                 http.StatusUnauthorized,
1033                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
1034         )
1035 }
1036
1037 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
1038         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1039                 arvadostest.FooCollection+".example.com/foo",
1040                 "?api_token=thisisabogustoken",
1041                 http.Header{
1042                         "Sec-Fetch-Mode": {"navigate"},
1043                         "Cache-Control":  {"no-cache"},
1044                 },
1045                 "",
1046                 http.StatusSeeOther,
1047                 "",
1048         )
1049         u, err := url.Parse(resp.Header().Get("Location"))
1050         c.Assert(err, check.IsNil)
1051         c.Logf("redirected to %s", u)
1052         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1053         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1054         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
1055 }
1056
1057 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
1058         for _, trial := range []struct {
1059                 anonToken    bool
1060                 cacheControl string
1061         }{
1062                 {},
1063                 {cacheControl: "no-cache"},
1064                 {anonToken: true},
1065                 {anonToken: true, cacheControl: "no-cache"},
1066         } {
1067                 c.Logf("trial: %+v", trial)
1068
1069                 if trial.anonToken {
1070                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1071                 } else {
1072                         s.handler.Cluster.Users.AnonymousUserToken = ""
1073                 }
1074                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1075                 c.Assert(err, check.IsNil)
1076                 req.Header.Set("Sec-Fetch-Mode", "navigate")
1077                 if trial.cacheControl != "" {
1078                         req.Header.Set("Cache-Control", trial.cacheControl)
1079                 }
1080                 resp := httptest.NewRecorder()
1081                 s.handler.ServeHTTP(resp, req)
1082                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
1083                 u, err := url.Parse(resp.Header().Get("Location"))
1084                 c.Assert(err, check.IsNil)
1085                 c.Logf("redirected to %q", u)
1086                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1087                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1088                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
1089         }
1090 }
1091
1092 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
1093         s.testVhostRedirectTokenToCookie(c, "GET",
1094                 "example.com/c="+arvadostest.FooCollection+"/foo",
1095                 "?api_token="+arvadostest.ActiveToken,
1096                 nil,
1097                 "",
1098                 http.StatusBadRequest,
1099                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
1100         )
1101 }
1102
1103 // If client requests an attachment by putting ?disposition=attachment
1104 // in the query string, and gets redirected, the redirect target
1105 // should respond with an attachment.
1106 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
1107         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1108                 arvadostest.FooCollection+".example.com/foo",
1109                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
1110                 nil,
1111                 "",
1112                 http.StatusOK,
1113                 "foo",
1114         )
1115         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1116 }
1117
1118 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
1119         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1120         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1121                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
1122                 "?api_token="+arvadostest.ActiveToken,
1123                 nil,
1124                 "",
1125                 http.StatusOK,
1126                 "foo",
1127         )
1128         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1129 }
1130
1131 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
1132         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1133         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1134                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
1135                 "?api_token="+arvadostest.ActiveToken,
1136                 nil,
1137                 "",
1138                 http.StatusOK,
1139                 "waz",
1140         )
1141         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1142         resp = s.testVhostRedirectTokenToCookie(c, "GET",
1143                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
1144                 "?api_token="+arvadostest.ActiveToken,
1145                 nil,
1146                 "",
1147                 http.StatusOK,
1148                 "waz",
1149         )
1150         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1151 }
1152
1153 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
1154         s.handler.Cluster.Collections.TrustAllContent = true
1155         s.testVhostRedirectTokenToCookie(c, "GET",
1156                 "example.com/c="+arvadostest.FooCollection+"/foo",
1157                 "?api_token="+arvadostest.ActiveToken,
1158                 nil,
1159                 "",
1160                 http.StatusOK,
1161                 "foo",
1162         )
1163 }
1164
1165 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
1166         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
1167
1168         s.testVhostRedirectTokenToCookie(c, "GET",
1169                 "example.com/c="+arvadostest.FooCollection+"/foo",
1170                 "?api_token="+arvadostest.ActiveToken,
1171                 nil,
1172                 "",
1173                 http.StatusBadRequest,
1174                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
1175         )
1176
1177         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1178                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
1179                 "?api_token="+arvadostest.ActiveToken,
1180                 nil,
1181                 "",
1182                 http.StatusOK,
1183                 "foo",
1184         )
1185         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
1186 }
1187
1188 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
1189         baseUrl := arvadostest.FooCollection + ".example.com/foo"
1190         query := url.Values{}
1191
1192         // The intent of these tests is to check that requests are redirected
1193         // correctly in the presence of multiple API tokens. The exact response
1194         // codes and content are not closely considered: they're just how
1195         // keep-web responded when we made the smallest possible fix. Changing
1196         // those responses may be okay, but you should still test all these
1197         // different cases and the associated redirect logic.
1198         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
1199         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1200         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
1201         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1202         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
1203         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1204         query["api_token"] = []string{"", arvadostest.ActiveToken}
1205         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1206
1207         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
1208         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
1209         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1210         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
1211         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1212         query["api_token"] = []string{"", arvadostest.AnonymousToken}
1213         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1214 }
1215
1216 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
1217         s.testVhostRedirectTokenToCookie(c, "POST",
1218                 arvadostest.FooCollection+".example.com/foo",
1219                 "",
1220                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
1221                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
1222                 http.StatusOK,
1223                 "foo",
1224         )
1225 }
1226
1227 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
1228         s.testVhostRedirectTokenToCookie(c, "POST",
1229                 arvadostest.FooCollection+".example.com/foo",
1230                 "",
1231                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
1232                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
1233                 http.StatusNotFound,
1234                 regexp.QuoteMeta(notFoundMessage+"\n"),
1235         )
1236 }
1237
1238 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
1239         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1240         s.testVhostRedirectTokenToCookie(c, "GET",
1241                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
1242                 "",
1243                 nil,
1244                 "",
1245                 http.StatusOK,
1246                 "Hello world\n",
1247         )
1248 }
1249
1250 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
1251         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
1252         s.testVhostRedirectTokenToCookie(c, "GET",
1253                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
1254                 "",
1255                 nil,
1256                 "",
1257                 http.StatusUnauthorized,
1258                 "Authorization tokens are not accepted here: .*\n",
1259         )
1260 }
1261
1262 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
1263         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1264
1265         client := arvados.NewClientFromEnv()
1266         client.AuthToken = arvadostest.ActiveToken
1267         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
1268         c.Assert(err, check.IsNil)
1269         path := `https:\\"odd' path chars`
1270         f, err := fs.OpenFile(path, os.O_CREATE, 0777)
1271         c.Assert(err, check.IsNil)
1272         f.Close()
1273         mtxt, err := fs.MarshalManifest(".")
1274         c.Assert(err, check.IsNil)
1275         var coll arvados.Collection
1276         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1277                 "collection": map[string]string{
1278                         "manifest_text": mtxt,
1279                 },
1280         })
1281         c.Assert(err, check.IsNil)
1282
1283         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
1284         req := &http.Request{
1285                 Method:     "GET",
1286                 Host:       u.Host,
1287                 URL:        u,
1288                 RequestURI: u.RequestURI(),
1289                 Header: http.Header{
1290                         "Authorization": {"Bearer " + client.AuthToken},
1291                 },
1292         }
1293         resp := httptest.NewRecorder()
1294         s.handler.ServeHTTP(resp, req)
1295         c.Check(resp.Code, check.Equals, http.StatusOK)
1296         doc, err := html.Parse(resp.Body)
1297         c.Assert(err, check.IsNil)
1298         pathHrefMap := getPathHrefMap(doc)
1299         c.Check(pathHrefMap, check.HasLen, 1) // the one leaf added to collection
1300         href, hasPath := pathHrefMap[path]
1301         c.Assert(hasPath, check.Equals, true) // the path is listed
1302         relUrl := mustParseURL(href)
1303         c.Check(relUrl.Path, check.Equals, "./"+path) // href can be decoded back to path
1304 }
1305
1306 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
1307         arv := arvados.NewClientFromEnv()
1308         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1309         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
1310         name := "foo/bar/baz"
1311         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
1312
1313         client := arvados.NewClientFromEnv()
1314         client.AuthToken = arvadostest.ActiveToken
1315         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
1316         c.Assert(err, check.IsNil)
1317         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
1318         c.Assert(err, check.IsNil)
1319         f.Close()
1320         mtxt, err := fs.MarshalManifest(".")
1321         c.Assert(err, check.IsNil)
1322         var coll arvados.Collection
1323         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1324                 "collection": map[string]string{
1325                         "manifest_text": mtxt,
1326                         "name":          name,
1327                         "owner_uuid":    arvadostest.AProjectUUID,
1328                 },
1329         })
1330         c.Assert(err, check.IsNil)
1331         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1332
1333         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
1334         for tryURL, expectedAnchorText := range map[string]string{
1335                 base:                   nameShown + "/",
1336                 base + nameShown + "/": "filename",
1337         } {
1338                 u, _ := url.Parse(tryURL)
1339                 req := &http.Request{
1340                         Method:     "GET",
1341                         Host:       u.Host,
1342                         URL:        u,
1343                         RequestURI: u.RequestURI(),
1344                         Header: http.Header{
1345                                 "Authorization": {"Bearer " + client.AuthToken},
1346                         },
1347                 }
1348                 resp := httptest.NewRecorder()
1349                 s.handler.ServeHTTP(resp, req)
1350                 c.Check(resp.Code, check.Equals, http.StatusOK)
1351                 doc, err := html.Parse(resp.Body)
1352                 c.Assert(err, check.IsNil) // valid HTML
1353                 pathHrefMap := getPathHrefMap(doc)
1354                 href, hasExpected := pathHrefMap[expectedAnchorText]
1355                 c.Assert(hasExpected, check.Equals, true) // has expected anchor text
1356                 c.Assert(href, check.Not(check.Equals), "")
1357                 relUrl := mustParseURL(href)
1358                 c.Check(relUrl.Path, check.Equals, "./"+expectedAnchorText) // decoded href maps back to the anchor text
1359         }
1360 }
1361
1362 // XHRs can't follow redirect-with-cookie so they rely on method=POST
1363 // and disposition=attachment (telling us it's acceptable to respond
1364 // with content instead of a redirect) and an Origin header that gets
1365 // added automatically by the browser (telling us it's desirable to do
1366 // so).
1367 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
1368         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
1369         req := &http.Request{
1370                 Method:     "POST",
1371                 Host:       u.Host,
1372                 URL:        u,
1373                 RequestURI: u.RequestURI(),
1374                 Header: http.Header{
1375                         "Origin":       {"https://origin.example"},
1376                         "Content-Type": {"application/x-www-form-urlencoded"},
1377                 },
1378                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1379                         "api_token":   {arvadostest.ActiveToken},
1380                         "disposition": {"attachment"},
1381                 }.Encode())),
1382         }
1383         resp := httptest.NewRecorder()
1384         s.handler.ServeHTTP(resp, req)
1385         c.Check(resp.Code, check.Equals, http.StatusOK)
1386         c.Check(resp.Body.String(), check.Equals, "foo")
1387         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1388
1389         // GET + Origin header is representative of both AJAX GET
1390         // requests and inline images via <IMG crossorigin="anonymous"
1391         // src="...">.
1392         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1393         req = &http.Request{
1394                 Method:     "GET",
1395                 Host:       u.Host,
1396                 URL:        u,
1397                 RequestURI: u.RequestURI(),
1398                 Header: http.Header{
1399                         "Origin": {"https://origin.example"},
1400                 },
1401         }
1402         resp = httptest.NewRecorder()
1403         s.handler.ServeHTTP(resp, req)
1404         c.Check(resp.Code, check.Equals, http.StatusOK)
1405         c.Check(resp.Body.String(), check.Equals, "foo")
1406         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1407 }
1408
1409 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1410         if reqHeader == nil {
1411                 reqHeader = http.Header{}
1412         }
1413         u, _ := url.Parse(`http://` + hostPath + queryString)
1414         c.Logf("requesting %s", u)
1415         req := &http.Request{
1416                 Method:     method,
1417                 Host:       u.Host,
1418                 URL:        u,
1419                 RequestURI: u.RequestURI(),
1420                 Header:     reqHeader,
1421                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1422         }
1423
1424         resp := httptest.NewRecorder()
1425         defer func() {
1426                 c.Check(resp.Code, check.Equals, expectStatus)
1427                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1428         }()
1429
1430         s.handler.ServeHTTP(resp, req)
1431         if resp.Code != http.StatusSeeOther {
1432                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1433                 // Since we're not redirecting, check that any api_token in the URL is
1434                 // handled safely.
1435                 // If there is no token in the URL, then we're good.
1436                 // Otherwise, if the response code is an error, the body is expected to
1437                 // be static content, and nothing that might maliciously introspect the
1438                 // URL. It's considered safe and allowed.
1439                 // Otherwise, if the response content has attachment disposition,
1440                 // that's considered safe for all the reasons explained in the
1441                 // safeAttachment comment in handler.go.
1442                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1443                 return resp
1444         }
1445
1446         loc, err := url.Parse(resp.Header().Get("Location"))
1447         c.Assert(err, check.IsNil)
1448         c.Check(loc.Scheme, check.Equals, u.Scheme)
1449         c.Check(loc.Host, check.Equals, u.Host)
1450         c.Check(loc.RawPath, check.Equals, u.RawPath)
1451         // If the response was a redirect, it should never include an API token.
1452         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1453         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1454         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1455
1456         c.Logf("following redirect to %s", u)
1457         req = &http.Request{
1458                 Method:     "GET",
1459                 Host:       loc.Host,
1460                 URL:        loc,
1461                 RequestURI: loc.RequestURI(),
1462                 Header:     reqHeader,
1463         }
1464         for _, c := range cookies {
1465                 req.AddCookie(c)
1466         }
1467
1468         resp = httptest.NewRecorder()
1469         s.handler.ServeHTTP(resp, req)
1470
1471         if resp.Code != http.StatusSeeOther {
1472                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1473         }
1474         return resp
1475 }
1476
1477 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1478         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1479         s.testDirectoryListing(c)
1480 }
1481
1482 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1483         s.handler.Cluster.Users.AnonymousUserToken = ""
1484         s.testDirectoryListing(c)
1485 }
1486
1487 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1488         // The "ownership cycle" test fixtures are reachable from the
1489         // "filter group without filters" group, causing webdav's
1490         // walkfs to recurse indefinitely. Avoid that by deleting one
1491         // of the bogus fixtures.
1492         arv := arvados.NewClientFromEnv()
1493         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1494         if err != nil {
1495                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1496                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1497         }
1498
1499         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1500         authHeader := http.Header{
1501                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1502         }
1503         for _, trial := range []struct {
1504                 uri      string
1505                 header   http.Header
1506                 expect   []string
1507                 redirect string
1508                 cutDirs  int
1509         }{
1510                 {
1511                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1512                         header:  authHeader,
1513                         expect:  []string{"dir1/foo", "dir1/bar"},
1514                         cutDirs: 0,
1515                 },
1516                 {
1517                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1518                         header:  authHeader,
1519                         expect:  []string{"foo", "bar"},
1520                         cutDirs: 1,
1521                 },
1522                 {
1523                         // URLs of this form ignore authHeader, and
1524                         // FooAndBarFilesInDirUUID isn't public, so
1525                         // this returns 401.
1526                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1527                         header: authHeader,
1528                         expect: nil,
1529                 },
1530                 {
1531                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1532                         header:  authHeader,
1533                         expect:  []string{"dir1/"},
1534                         cutDirs: 3,
1535                 },
1536                 {
1537                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1538                         header:  authHeader,
1539                         expect:  []string{"bar"},
1540                         cutDirs: 4,
1541                 },
1542                 {
1543                         uri:     "download.example.com/",
1544                         header:  authHeader,
1545                         expect:  []string{"users/"},
1546                         cutDirs: 0,
1547                 },
1548                 {
1549                         uri:      "download.example.com/users",
1550                         header:   authHeader,
1551                         redirect: "/users/",
1552                         expect:   []string{"active/"},
1553                         cutDirs:  1,
1554                 },
1555                 {
1556                         uri:     "download.example.com/users/",
1557                         header:  authHeader,
1558                         expect:  []string{"active/"},
1559                         cutDirs: 1,
1560                 },
1561                 {
1562                         uri:      "download.example.com/users/active",
1563                         header:   authHeader,
1564                         redirect: "/users/active/",
1565                         expect:   []string{"foo_file_in_dir/"},
1566                         cutDirs:  2,
1567                 },
1568                 {
1569                         uri:     "download.example.com/users/active/",
1570                         header:  authHeader,
1571                         expect:  []string{"foo_file_in_dir/"},
1572                         cutDirs: 2,
1573                 },
1574                 {
1575                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1576                         header:  nil,
1577                         expect:  []string{"dir1/foo", "dir1/bar"},
1578                         cutDirs: 4,
1579                 },
1580                 {
1581                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1582                         header:  nil,
1583                         expect:  []string{"dir1/foo", "dir1/bar"},
1584                         cutDirs: 2,
1585                 },
1586                 {
1587                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1588                         header:  nil,
1589                         expect:  []string{"dir1/foo", "dir1/bar"},
1590                         cutDirs: 2,
1591                 },
1592                 {
1593                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1594                         header:  authHeader,
1595                         expect:  []string{"dir1/foo", "dir1/bar"},
1596                         cutDirs: 1,
1597                 },
1598                 {
1599                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1600                         header:   authHeader,
1601                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1602                         expect:   []string{"foo", "bar"},
1603                         cutDirs:  2,
1604                 },
1605                 {
1606                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1607                         header:  authHeader,
1608                         expect:  []string{"foo", "bar"},
1609                         cutDirs: 3,
1610                 },
1611                 {
1612                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1613                         header:   authHeader,
1614                         redirect: "/dir1/",
1615                         expect:   []string{"foo", "bar"},
1616                         cutDirs:  1,
1617                 },
1618                 {
1619                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1620                         header: authHeader,
1621                         expect: nil,
1622                 },
1623                 {
1624                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1625                         header:  authHeader,
1626                         expect:  []string{"waz"},
1627                         cutDirs: 1,
1628                 },
1629                 {
1630                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1631                         header:  authHeader,
1632                         expect:  []string{"waz"},
1633                         cutDirs: 2,
1634                 },
1635                 {
1636                         uri:     "download.example.com/users/active/This filter group/",
1637                         header:  authHeader,
1638                         expect:  []string{"A Subproject/"},
1639                         cutDirs: 3,
1640                 },
1641                 {
1642                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1643                         header:  authHeader,
1644                         expect:  []string{"baz_file/"},
1645                         cutDirs: 4,
1646                 },
1647                 {
1648                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1649                         header:  authHeader,
1650                         expect:  []string{"A Subproject/"},
1651                         cutDirs: 2,
1652                 },
1653                 {
1654                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1655                         header:  authHeader,
1656                         expect:  []string{"baz_file/"},
1657                         cutDirs: 3,
1658                 },
1659         } {
1660                 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1661                 resp := httptest.NewRecorder()
1662                 u := mustParseURL("//" + trial.uri)
1663                 req := &http.Request{
1664                         Method:     "GET",
1665                         Host:       u.Host,
1666                         URL:        u,
1667                         RequestURI: u.RequestURI(),
1668                         Header:     copyHeader(trial.header),
1669                 }
1670                 s.handler.ServeHTTP(resp, req)
1671                 var cookies []*http.Cookie
1672                 for resp.Code == http.StatusSeeOther {
1673                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1674                         req = &http.Request{
1675                                 Method:     "GET",
1676                                 Host:       u.Host,
1677                                 URL:        u,
1678                                 RequestURI: u.RequestURI(),
1679                                 Header:     copyHeader(trial.header),
1680                         }
1681                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1682                         for _, c := range cookies {
1683                                 req.AddCookie(c)
1684                         }
1685                         resp = httptest.NewRecorder()
1686                         s.handler.ServeHTTP(resp, req)
1687                 }
1688                 if trial.redirect != "" {
1689                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1690                 }
1691                 if trial.expect == nil {
1692                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1693                 } else {
1694                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1695                         listingPageDoc, err := html.Parse(resp.Body)
1696                         c.Check(err, check.IsNil, comment) // valid HTML document
1697                         pathHrefMap := getPathHrefMap(listingPageDoc)
1698                         c.Assert(pathHrefMap, check.Not(check.HasLen), 0, comment)
1699                         for _, e := range trial.expect {
1700                                 href, hasE := pathHrefMap[e]
1701                                 c.Check(hasE, check.Equals, true, comment) // expected path is listed
1702                                 relUrl := mustParseURL(href)
1703                                 c.Check(relUrl.Path, check.Equals, "./"+e, comment) // href can be decoded back to path
1704                         }
1705                         wgetCommand := getWgetExamplePre(listingPageDoc)
1706                         wgetExpected := regexp.MustCompile(`^\$ wget .*--cut-dirs=(\d+) .*'(https?://[^']+)'$`)
1707                         wgetMatchGroups := wgetExpected.FindStringSubmatch(wgetCommand)
1708                         c.Assert(wgetMatchGroups, check.NotNil)                                     // wget command matches
1709                         c.Check(wgetMatchGroups[1], check.Equals, fmt.Sprintf("%d", trial.cutDirs)) // correct level of cut dirs in wget command
1710                         printedUrl := mustParseURL(wgetMatchGroups[2])
1711                         c.Check(printedUrl.Host, check.Equals, req.URL.Host)
1712                         c.Check(printedUrl.Path, check.Equals, req.URL.Path) // URL arg in wget command can be decoded to the right path
1713                 }
1714
1715                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1716                 req = &http.Request{
1717                         Method:     "OPTIONS",
1718                         Host:       u.Host,
1719                         URL:        u,
1720                         RequestURI: u.RequestURI(),
1721                         Header:     copyHeader(trial.header),
1722                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1723                 }
1724                 resp = httptest.NewRecorder()
1725                 s.handler.ServeHTTP(resp, req)
1726                 if trial.expect == nil {
1727                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1728                 } else {
1729                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1730                 }
1731
1732                 req = &http.Request{
1733                         Method:     "PROPFIND",
1734                         Host:       u.Host,
1735                         URL:        u,
1736                         RequestURI: u.RequestURI(),
1737                         Header:     copyHeader(trial.header),
1738                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1739                 }
1740                 resp = httptest.NewRecorder()
1741                 s.handler.ServeHTTP(resp, req)
1742                 // This check avoids logging a big XML document in the
1743                 // event webdav throws a 500 error after sending
1744                 // headers for a 207.
1745                 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1746                         continue
1747                 }
1748                 if trial.expect == nil {
1749                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1750                 } else {
1751                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1752                         for _, e := range trial.expect {
1753                                 if strings.HasSuffix(e, "/") {
1754                                         e = filepath.Join(u.Path, e) + "/"
1755                                 } else {
1756                                         e = filepath.Join(u.Path, e)
1757                                 }
1758                                 e = strings.Replace(e, " ", "%20", -1)
1759                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1760                         }
1761                 }
1762         }
1763 }
1764
1765 // Shallow-traverse the HTML document, gathering the nodes satisfying the
1766 // predicate function in the output slice. If a node matches the predicate,
1767 // none of its children will be visited.
1768 func getNodes(document *html.Node, predicate func(*html.Node) bool) []*html.Node {
1769         var acc []*html.Node
1770         var traverse func(*html.Node, []*html.Node) []*html.Node
1771         traverse = func(root *html.Node, sofar []*html.Node) []*html.Node {
1772                 if root == nil {
1773                         return sofar
1774                 }
1775                 if predicate(root) {
1776                         return append(sofar, root)
1777                 }
1778                 for cur := root.FirstChild; cur != nil; cur = cur.NextSibling {
1779                         sofar = traverse(cur, sofar)
1780                 }
1781                 return sofar
1782         }
1783         return traverse(document, acc)
1784 }
1785
1786 // Returns true if a node has the attribute targetAttr with the given value
1787 func matchesAttributeValue(node *html.Node, targetAttr string, value string) bool {
1788         for _, attr := range node.Attr {
1789                 if attr.Key == targetAttr && attr.Val == value {
1790                         return true
1791                 }
1792         }
1793         return false
1794 }
1795
1796 // Concatenate the content of text-node children of node; only direct
1797 // children are visited, and any non-text children are skipped.
1798 func getNodeText(node *html.Node) string {
1799         var recv strings.Builder
1800         for c := node.FirstChild; c != nil; c = c.NextSibling {
1801                 if c.Type == html.TextNode {
1802                         recv.WriteString(c.Data)
1803                 }
1804         }
1805         return recv.String()
1806 }
1807
1808 // Returns a map from the directory listing item string (a path) to the href
1809 // value of its <a> tag (an encoded relative URL)
1810 func getPathHrefMap(document *html.Node) map[string]string {
1811         isItemATag := func(node *html.Node) bool {
1812                 return node.Type == html.ElementNode && node.Data == "a" && matchesAttributeValue(node, "class", "item")
1813         }
1814         aTags := getNodes(document, isItemATag)
1815         output := make(map[string]string)
1816         for _, elem := range aTags {
1817                 textContent := getNodeText(elem)
1818                 for _, attr := range elem.Attr {
1819                         if attr.Key == "href" {
1820                                 output[textContent] = attr.Val
1821                                 break
1822                         }
1823                 }
1824         }
1825         return output
1826 }
1827
1828 func getWgetExamplePre(document *html.Node) string {
1829         isWgetPre := func(node *html.Node) bool {
1830                 return node.Type == html.ElementNode && matchesAttributeValue(node, "id", "wget-example")
1831         }
1832         elements := getNodes(document, isWgetPre)
1833         if len(elements) != 1 {
1834                 return ""
1835         }
1836         return getNodeText(elements[0])
1837 }
1838
1839 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1840         arv := arvados.NewClientFromEnv()
1841         var newCollection arvados.Collection
1842         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1843                 "collection": map[string]string{
1844                         "owner_uuid":    arvadostest.ActiveUserUUID,
1845                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1846                         "name":          "keep-web test collection",
1847                 },
1848                 "ensure_unique_name": true,
1849         })
1850         c.Assert(err, check.IsNil)
1851         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1852
1853         var updated arvados.Collection
1854         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1855                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1856                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1857                 req := &http.Request{
1858                         Method:     "DELETE",
1859                         Host:       u.Host,
1860                         URL:        u,
1861                         RequestURI: u.RequestURI(),
1862                         Header: http.Header{
1863                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1864                         },
1865                 }
1866                 resp := httptest.NewRecorder()
1867                 s.handler.ServeHTTP(resp, req)
1868                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1869
1870                 updated = arvados.Collection{}
1871                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1872                 c.Check(err, check.IsNil)
1873                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1874                 c.Logf("updated manifest_text %q", updated.ManifestText)
1875         }
1876         c.Check(updated.ManifestText, check.Equals, "")
1877 }
1878
1879 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1880         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1881
1882         client := arvados.NewClientFromEnv()
1883         client.AuthToken = arvadostest.ActiveToken
1884         arv, err := arvadosclient.New(client)
1885         c.Assert(err, check.Equals, nil)
1886         kc, err := keepclient.MakeKeepClient(arv)
1887         c.Assert(err, check.Equals, nil)
1888
1889         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1890         c.Assert(err, check.IsNil)
1891
1892         trials := []struct {
1893                 filename    string
1894                 content     string
1895                 contentType string
1896         }{
1897                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1898                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1899                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1900                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1901                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1902         }
1903         for _, trial := range trials {
1904                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1905                 c.Assert(err, check.IsNil)
1906                 _, err = f.Write([]byte(trial.content))
1907                 c.Assert(err, check.IsNil)
1908                 c.Assert(f.Close(), check.IsNil)
1909         }
1910         mtxt, err := fs.MarshalManifest(".")
1911         c.Assert(err, check.IsNil)
1912         var coll arvados.Collection
1913         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1914                 "collection": map[string]string{
1915                         "manifest_text": mtxt,
1916                 },
1917         })
1918         c.Assert(err, check.IsNil)
1919
1920         for _, trial := range trials {
1921                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1922                 req := &http.Request{
1923                         Method:     "GET",
1924                         Host:       u.Host,
1925                         URL:        u,
1926                         RequestURI: u.RequestURI(),
1927                         Header: http.Header{
1928                                 "Authorization": {"Bearer " + client.AuthToken},
1929                         },
1930                 }
1931                 resp := httptest.NewRecorder()
1932                 s.handler.ServeHTTP(resp, req)
1933                 c.Check(resp.Code, check.Equals, http.StatusOK)
1934                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1935                 c.Check(resp.Body.String(), check.Equals, trial.content)
1936         }
1937 }
1938
1939 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1940         req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1941         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1942         c.Assert(err, check.IsNil)
1943         resp := httptest.NewRecorder()
1944         s.handler.ServeHTTP(resp, req)
1945         c.Assert(resp.Code, check.Equals, http.StatusOK)
1946         c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1947 }
1948
1949 // Writing to a collection shouldn't affect its entry in the
1950 // PDH-to-manifest cache.
1951 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1952         arv, err := arvadosclient.MakeArvadosClient()
1953         c.Assert(err, check.Equals, nil)
1954         arv.ApiToken = arvadostest.ActiveToken
1955
1956         u := mustParseURL("http://x.example/testfile")
1957         req := &http.Request{
1958                 Method:     "GET",
1959                 Host:       u.Host,
1960                 URL:        u,
1961                 RequestURI: u.RequestURI(),
1962                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1963         }
1964
1965         checkWithID := func(id string, status int) {
1966                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1967                 req.Host = req.URL.Host
1968                 resp := httptest.NewRecorder()
1969                 s.handler.ServeHTTP(resp, req)
1970                 c.Check(resp.Code, check.Equals, status)
1971         }
1972
1973         var colls [2]arvados.Collection
1974         for i := range colls {
1975                 err := arv.Create("collections",
1976                         map[string]interface{}{
1977                                 "ensure_unique_name": true,
1978                                 "collection": map[string]interface{}{
1979                                         "name": "test collection",
1980                                 },
1981                         }, &colls[i])
1982                 c.Assert(err, check.Equals, nil)
1983         }
1984
1985         // Populate cache with empty collection
1986         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1987
1988         // write a file to colls[0]
1989         reqPut := *req
1990         reqPut.Method = "PUT"
1991         reqPut.URL.Host = colls[0].UUID + ".example"
1992         reqPut.Host = req.URL.Host
1993         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1994         resp := httptest.NewRecorder()
1995         s.handler.ServeHTTP(resp, &reqPut)
1996         c.Check(resp.Code, check.Equals, http.StatusCreated)
1997
1998         // new file should not appear in colls[1]
1999         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
2000         checkWithID(colls[1].UUID, http.StatusNotFound)
2001
2002         checkWithID(colls[0].UUID, http.StatusOK)
2003 }
2004
2005 func copyHeader(h http.Header) http.Header {
2006         hc := http.Header{}
2007         for k, v := range h {
2008                 hc[k] = append([]string(nil), v...)
2009         }
2010         return hc
2011 }
2012
2013 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
2014         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
2015
2016         client := arvados.NewClientFromEnv()
2017         client.AuthToken = arvadostest.AdminToken
2018         var logentries arvados.LogList
2019         limit1 := 1
2020         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
2021                 arvados.ResourceListParams{
2022                         Limit: &limit1,
2023                         Order: "created_at desc"})
2024         c.Check(err, check.IsNil)
2025         c.Check(logentries.Items, check.HasLen, 1)
2026         lastLogId := logentries.Items[0].ID
2027         c.Logf("lastLogId: %d", lastLogId)
2028
2029         var logbuf bytes.Buffer
2030         logger := logrus.New()
2031         logger.Out = &logbuf
2032         resp := httptest.NewRecorder()
2033         req = req.WithContext(ctxlog.Context(context.Background(), logger))
2034         s.handler.ServeHTTP(resp, req)
2035
2036         if perm {
2037                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
2038                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
2039                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
2040
2041                 deadline := time.Now().Add(time.Second)
2042                 for {
2043                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
2044                         logentries = arvados.LogList{}
2045                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
2046                                 arvados.ResourceListParams{
2047                                         Filters: []arvados.Filter{
2048                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
2049                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
2050                                         },
2051                                         Limit: &limit1,
2052                                         Order: "created_at desc",
2053                                 })
2054                         c.Assert(err, check.IsNil)
2055                         if len(logentries.Items) > 0 &&
2056                                 logentries.Items[0].ID > lastLogId &&
2057                                 logentries.Items[0].ObjectUUID == userUuid &&
2058                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
2059                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
2060                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
2061                                 break
2062                         }
2063                         c.Logf("logentries.Items: %+v", logentries.Items)
2064                         time.Sleep(50 * time.Millisecond)
2065                 }
2066         } else {
2067                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
2068                 c.Check(logbuf.String(), check.Equals, "")
2069         }
2070 }
2071
2072 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
2073         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
2074
2075         s.handler.Cluster.Collections.TrustAllContent = true
2076         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(0)
2077
2078         for _, adminperm := range []bool{true, false} {
2079                 for _, userperm := range []bool{true, false} {
2080                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
2081                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
2082
2083                         // Test admin permission
2084                         req := &http.Request{
2085                                 Method:     "GET",
2086                                 Host:       u.Host,
2087                                 URL:        u,
2088                                 RequestURI: u.RequestURI(),
2089                                 Header: http.Header{
2090                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
2091                                 },
2092                         }
2093                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
2094                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
2095
2096                         // Test user permission
2097                         req = &http.Request{
2098                                 Method:     "GET",
2099                                 Host:       u.Host,
2100                                 URL:        u,
2101                                 RequestURI: u.RequestURI(),
2102                                 Header: http.Header{
2103                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2104                                 },
2105                         }
2106                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
2107                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
2108                 }
2109         }
2110
2111         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
2112
2113         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
2114                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
2115
2116                 u = mustParseURL(tryurl)
2117                 req := &http.Request{
2118                         Method:     "GET",
2119                         Host:       u.Host,
2120                         URL:        u,
2121                         RequestURI: u.RequestURI(),
2122                         Header: http.Header{
2123                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
2124                         },
2125                 }
2126                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
2127                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
2128         }
2129
2130         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
2131         req := &http.Request{
2132                 Method:     "GET",
2133                 Host:       u.Host,
2134                 URL:        u,
2135                 RequestURI: u.RequestURI(),
2136                 Header: http.Header{
2137                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2138                 },
2139         }
2140         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
2141                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
2142 }
2143
2144 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
2145         for _, adminperm := range []bool{true, false} {
2146                 for _, userperm := range []bool{true, false} {
2147
2148                         arv := arvados.NewClientFromEnv()
2149                         arv.AuthToken = arvadostest.ActiveToken
2150
2151                         var coll arvados.Collection
2152                         err := arv.RequestAndDecode(&coll,
2153                                 "POST",
2154                                 "/arvados/v1/collections",
2155                                 nil,
2156                                 map[string]interface{}{
2157                                         "ensure_unique_name": true,
2158                                         "collection": map[string]interface{}{
2159                                                 "name": "test collection",
2160                                         },
2161                                 })
2162                         c.Assert(err, check.Equals, nil)
2163
2164                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
2165
2166                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
2167                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
2168
2169                         // Test admin permission
2170                         req := &http.Request{
2171                                 Method:     "PUT",
2172                                 Host:       u.Host,
2173                                 URL:        u,
2174                                 RequestURI: u.RequestURI(),
2175                                 Header: http.Header{
2176                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
2177                                 },
2178                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
2179                         }
2180                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
2181                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
2182
2183                         // Test user permission
2184                         req = &http.Request{
2185                                 Method:     "PUT",
2186                                 Host:       u.Host,
2187                                 URL:        u,
2188                                 RequestURI: u.RequestURI(),
2189                                 Header: http.Header{
2190                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2191                                 },
2192                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
2193                         }
2194                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
2195                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
2196                 }
2197         }
2198 }
2199
2200 func (s *IntegrationSuite) serveAndLogRequests(c *check.C, reqs *map[*http.Request]int) *bytes.Buffer {
2201         logbuf, ctx := newLoggerAndContext()
2202         var wg sync.WaitGroup
2203         for req, expectStatus := range *reqs {
2204                 req := req.WithContext(ctx)
2205                 expectStatus := expectStatus
2206                 wg.Add(1)
2207                 go func() {
2208                         defer wg.Done()
2209                         resp := httptest.NewRecorder()
2210                         s.handler.ServeHTTP(resp, req)
2211                         c.Check(resp.Result().StatusCode, check.Equals, expectStatus)
2212                 }()
2213         }
2214         wg.Wait()
2215         return logbuf
2216 }
2217
2218 func countLogMatches(c *check.C, logbuf *bytes.Buffer, pattern string, matchCount int) bool {
2219         search, err := regexp.Compile(pattern)
2220         if !c.Check(err, check.IsNil, check.Commentf("failed to compile regexp: %v", err)) {
2221                 return false
2222         }
2223         matches := search.FindAll(logbuf.Bytes(), -1)
2224         return c.Check(matches, check.HasLen, matchCount,
2225                 check.Commentf("%d matching log messages: %+v", len(matches), matches))
2226 }
2227
2228 func (s *IntegrationSuite) TestLogThrottling(c *check.C) {
2229         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Hour)
2230         fooURL := "http://" + arvadostest.FooCollection + ".keep-web.example/foo"
2231         req := newRequest("GET", fooURL)
2232         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
2233         pattern := `\bmsg="File download".* collection_file_path=foo\b`
2234
2235         // All these requests get byte zero and should be logged.
2236         reqs := make(map[*http.Request]int)
2237         reqs[req] = http.StatusOK
2238         for _, byterange := range []string{"0-2", "0-1", "0-", "-3"} {
2239                 req := req.Clone(context.Background())
2240                 req.Header.Set("Range", "bytes="+byterange)
2241                 reqs[req] = http.StatusPartialContent
2242         }
2243         logbuf := s.serveAndLogRequests(c, &reqs)
2244         countLogMatches(c, logbuf, pattern, len(reqs))
2245
2246         // None of these requests get byte zero so they should all be throttled
2247         // (now that we've made at least one request for byte zero).
2248         reqs = make(map[*http.Request]int)
2249         for _, byterange := range []string{"1-2", "1-", "2-", "-1", "-2"} {
2250                 req := req.Clone(context.Background())
2251                 req.Header.Set("Range", "bytes="+byterange)
2252                 reqs[req] = http.StatusPartialContent
2253         }
2254         logbuf = s.serveAndLogRequests(c, &reqs)
2255         countLogMatches(c, logbuf, pattern, 0)
2256 }
2257
2258 func (s *IntegrationSuite) TestLogThrottleInterval(c *check.C) {
2259         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Nanosecond)
2260         logbuf, ctx := newLoggerAndContext()
2261         req := newRequest("GET", "http://"+arvadostest.FooCollection+".keep-web.example/foo")
2262         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
2263         req = req.WithContext(ctx)
2264
2265         re := regexp.MustCompile(`\bmsg="File download".* collection_file_path=foo\b`)
2266         for expected := 1; expected < 4; expected++ {
2267                 time.Sleep(2 * time.Nanosecond)
2268                 resp := httptest.NewRecorder()
2269                 s.handler.ServeHTTP(resp, req)
2270                 c.Assert(resp.Result().StatusCode, check.Equals, http.StatusOK)
2271                 matches := re.FindAll(logbuf.Bytes(), -1)
2272                 c.Assert(matches, check.HasLen, expected,
2273                         check.Commentf("%d matching log messages: %+v", len(matches), matches))
2274         }
2275 }
2276
2277 func (s *IntegrationSuite) TestLogThrottleDifferentTokens(c *check.C) {
2278         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Hour)
2279         req := newRequest("GET", "http://"+arvadostest.FooCollection+".keep-web.example/foo")
2280         reqs := make(map[*http.Request]int)
2281         for _, token := range []string{arvadostest.ActiveToken, arvadostest.AdminToken} {
2282                 req := req.Clone(context.Background())
2283                 req.Header.Set("Authorization", "Bearer "+token)
2284                 reqs[req] = http.StatusOK
2285         }
2286         logbuf := s.serveAndLogRequests(c, &reqs)
2287         countLogMatches(c, logbuf, `\bmsg="File download".* collection_file_path=foo\b`, len(reqs))
2288 }
2289
2290 func (s *IntegrationSuite) TestLogThrottleDifferentFiles(c *check.C) {
2291         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Hour)
2292         baseURL := "http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/"
2293         reqs := make(map[*http.Request]int)
2294         for _, filename := range []string{"file1", "file2", "file3"} {
2295                 req := newRequest("GET", baseURL+filename)
2296                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
2297                 reqs[req] = http.StatusOK
2298         }
2299         logbuf := s.serveAndLogRequests(c, &reqs)
2300         countLogMatches(c, logbuf, `\bmsg="File download".* collection_uuid=`+arvadostest.MultilevelCollection1+`\b`, len(reqs))
2301 }
2302
2303 func (s *IntegrationSuite) TestLogThrottleDifferentSources(c *check.C) {
2304         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Hour)
2305         req := newRequest("GET", "http://"+arvadostest.FooCollection+".keep-web.example/foo")
2306         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
2307         reqs := make(map[*http.Request]int)
2308         reqs[req] = http.StatusOK
2309         for _, xff := range []string{"10.22.33.44", "100::123"} {
2310                 req := req.Clone(context.Background())
2311                 req.Header.Set("X-Forwarded-For", xff)
2312                 reqs[req] = http.StatusOK
2313         }
2314         logbuf := s.serveAndLogRequests(c, &reqs)
2315         countLogMatches(c, logbuf, `\bmsg="File download".* collection_file_path=foo\b`, len(reqs))
2316 }
2317
2318 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
2319         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
2320         client := arvados.NewClientFromEnv()
2321         client.AuthToken = arvadostest.ActiveTokenV2
2322         var handler http.Handler = s.handler
2323         // handler = httpserver.AddRequestIDs(httpserver.LogRequests(s.handler)) // ...to enable request logging in test output
2324
2325         // Each file we upload will consist of some unique content
2326         // followed by 2 MiB of filler content.
2327         filler := "."
2328         for i := 0; i < 21; i++ {
2329                 filler += filler
2330         }
2331
2332         // Start small, and increase concurrency (2^2, 4^2, ...)
2333         // only until hitting failure. Avoids unnecessarily long
2334         // failure reports.
2335         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
2336                 c.Logf("%s: n=%d", c.TestName(), n)
2337
2338                 var coll arvados.Collection
2339                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
2340                 c.Assert(err, check.IsNil)
2341                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
2342
2343                 var wg sync.WaitGroup
2344                 for i := 0; i < n && !c.Failed(); i++ {
2345                         i := i
2346                         wg.Add(1)
2347                         go func() {
2348                                 defer wg.Done()
2349                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
2350                                 resp := httptest.NewRecorder()
2351                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
2352                                 c.Assert(err, check.IsNil)
2353                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2354                                 handler.ServeHTTP(resp, req)
2355                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
2356                                 for j := 0; j < n && !c.Failed(); j++ {
2357                                         j := j
2358                                         wg.Add(1)
2359                                         go func() {
2360                                                 defer wg.Done()
2361                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
2362                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
2363
2364                                                 resp := httptest.NewRecorder()
2365                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content+filler))
2366                                                 c.Assert(err, check.IsNil)
2367                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2368                                                 handler.ServeHTTP(resp, req)
2369                                                 c.Check(resp.Code, check.Equals, http.StatusCreated, check.Commentf("%s", content))
2370
2371                                                 time.Sleep(time.Second)
2372                                                 resp = httptest.NewRecorder()
2373                                                 req, err = http.NewRequest("GET", u.String(), nil)
2374                                                 c.Assert(err, check.IsNil)
2375                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2376                                                 handler.ServeHTTP(resp, req)
2377                                                 c.Check(resp.Code, check.Equals, http.StatusOK, check.Commentf("%s", content))
2378                                                 c.Check(strings.TrimSuffix(resp.Body.String(), filler), check.Equals, content)
2379                                         }()
2380                                 }
2381                         }()
2382                 }
2383                 wg.Wait()
2384                 for i := 0; i < n; i++ {
2385                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
2386                         resp := httptest.NewRecorder()
2387                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
2388                         c.Assert(err, check.IsNil)
2389                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2390                         s.handler.ServeHTTP(resp, req)
2391                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
2392                 }
2393         }
2394 }
2395
2396 func (s *IntegrationSuite) TestDepthHeader(c *check.C) {
2397         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
2398         client := arvados.NewClientFromEnv()
2399         client.AuthToken = arvadostest.ActiveTokenV2
2400
2401         var coll arvados.Collection
2402         err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
2403         c.Assert(err, check.IsNil)
2404         defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
2405         base := "http://" + coll.UUID + ".collections.example.com/"
2406
2407         for _, trial := range []struct {
2408                 method      string
2409                 path        string
2410                 destination string
2411                 depth       string
2412                 expectCode  int // 0 means expect 2xx
2413         }{
2414                 // setup...
2415                 {method: "MKCOL", path: "dir"},
2416                 {method: "PUT", path: "dir/file"},
2417                 {method: "MKCOL", path: "dir/dir2"},
2418                 // delete with no depth = OK
2419                 {method: "DELETE", path: "dir/dir2", depth: ""},
2420                 // delete with depth other than infinity = fail
2421                 {method: "DELETE", path: "dir", depth: "0", expectCode: 400},
2422                 {method: "DELETE", path: "dir", depth: "1", expectCode: 400},
2423                 // delete with depth infinity = OK
2424                 {method: "DELETE", path: "dir", depth: "infinity"},
2425
2426                 // setup...
2427                 {method: "MKCOL", path: "dir"},
2428                 {method: "PUT", path: "dir/file"},
2429                 {method: "MKCOL", path: "dir/dir2"},
2430                 // move with depth other than infinity = fail
2431                 {method: "MOVE", path: "dir", destination: "moved", depth: "0", expectCode: 400},
2432                 {method: "MOVE", path: "dir", destination: "moved", depth: "1", expectCode: 400},
2433                 // move with depth infinity = OK
2434                 {method: "MOVE", path: "dir", destination: "moved", depth: "infinity"},
2435                 {method: "DELETE", path: "moved"},
2436
2437                 // setup...
2438                 {method: "MKCOL", path: "dir"},
2439                 {method: "PUT", path: "dir/file"},
2440                 {method: "MKCOL", path: "dir/dir2"},
2441                 // copy with depth 0 = create empty destination dir
2442                 {method: "COPY", path: "dir/", destination: "copied-empty/", depth: "0"},
2443                 {method: "DELETE", path: "copied-empty/file", expectCode: 404},
2444                 {method: "DELETE", path: "copied-empty"},
2445                 // copy with depth 0 = create empty destination dir
2446                 // (destination dir has no trailing slash this time)
2447                 {method: "COPY", path: "dir/", destination: "copied-empty-noslash", depth: "0"},
2448                 {method: "DELETE", path: "copied-empty-noslash/file", expectCode: 404},
2449                 {method: "DELETE", path: "copied-empty-noslash"},
2450                 // copy with depth 0 = create empty destination dir
2451                 // (source dir has no trailing slash this time)
2452                 {method: "COPY", path: "dir", destination: "copied-empty-noslash", depth: "0"},
2453                 {method: "DELETE", path: "copied-empty-noslash/file", expectCode: 404},
2454                 {method: "DELETE", path: "copied-empty-noslash"},
2455                 // copy with depth 1 = fail
2456                 {method: "COPY", path: "dir", destination: "copied", depth: "1", expectCode: 400},
2457                 // copy with depth infinity = copy entire subtree
2458                 {method: "COPY", path: "dir/", destination: "copied", depth: "infinity"},
2459                 {method: "DELETE", path: "copied/file"},
2460                 {method: "DELETE", path: "copied"},
2461                 // copy with depth infinity = copy entire subtree
2462                 // (source dir has no trailing slash this time)
2463                 {method: "COPY", path: "dir", destination: "copied", depth: "infinity"},
2464                 {method: "DELETE", path: "copied/file"},
2465                 {method: "DELETE", path: "copied"},
2466                 // cleanup
2467                 {method: "DELETE", path: "dir"},
2468         } {
2469                 c.Logf("trial %+v", trial)
2470                 resp := httptest.NewRecorder()
2471                 req, err := http.NewRequest(trial.method, base+trial.path, strings.NewReader(""))
2472                 c.Assert(err, check.IsNil)
2473                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2474                 if trial.destination != "" {
2475                         req.Header.Set("Destination", base+trial.destination)
2476                 }
2477                 if trial.depth != "" {
2478                         req.Header.Set("Depth", trial.depth)
2479                 }
2480                 s.handler.ServeHTTP(resp, req)
2481                 if trial.expectCode != 0 {
2482                         c.Assert(resp.Code, check.Equals, trial.expectCode)
2483                 } else {
2484                         c.Assert(resp.Code >= 200, check.Equals, true, check.Commentf("got code %d", resp.Code))
2485                         c.Assert(resp.Code < 300, check.Equals, true, check.Commentf("got code %d", resp.Code))
2486                 }
2487                 c.Logf("resp.Body: %q", resp.Body.String())
2488         }
2489 }