services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "sync"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         "golang.org/x/net/html"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63                 metrics: newMetrics(prometheus.NewRegistry()),
  64         }
  65 }
  66
  67 func newCollection(collID string) *arvados.Collection {
  68         coll := &arvados.Collection{UUID: collID}
  69         if pdh, ok := arvadostest.TestCollectionUUIDToPDH[collID]; ok {
  70                 coll.PortableDataHash = pdh
  71         }
  72         return coll
  73 }
  74
  75 func newRequest(method, urlStr string) *http.Request {
  76         u := mustParseURL(urlStr)
  77         return &http.Request{
  78                 Method:     method,
  79                 Host:       u.Host,
  80                 URL:        u,
  81                 RequestURI: u.RequestURI(),
  82                 RemoteAddr: "10.20.30.40:56789",
  83                 Header:     http.Header{},
  84         }
  85 }
  86
  87 func newLoggerAndContext() (*bytes.Buffer, context.Context) {
  88         var logbuf bytes.Buffer
  89         logger := logrus.New()
  90         logger.Out = &logbuf
  91         return &logbuf, ctxlog.Context(context.Background(), logger)
  92 }
  93
  94 func (s *UnitSuite) TestLogEventTypes(c *check.C) {
  95         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
  96         for method, expected := range map[string]string{
  97                 "GET":  "file_download",
  98                 "POST": "file_upload",
  99                 "PUT":  "file_upload",
 100         } {
 101                 filePath := "/" + method
 102                 req := newRequest(method, collURL+filePath)
 103                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 104                 if !c.Check(actual, check.NotNil) {
 105                         continue
 106                 }
 107                 c.Check(actual.eventType, check.Equals, expected)
 108         }
 109 }
 110
 111 func (s *UnitSuite) TestUnloggedEventTypes(c *check.C) {
 112         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 113         for _, method := range []string{"DELETE", "HEAD", "OPTIONS", "PATCH"} {
 114                 filePath := "/" + method
 115                 req := newRequest(method, collURL+filePath)
 116                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 117                 c.Check(actual, check.IsNil,
 118                         check.Commentf("%s request made a log event", method))
 119         }
 120 }
 121
 122 func (s *UnitSuite) TestLogFilePath(c *check.C) {
 123         coll := newCollection(arvadostest.FooCollection)
 124         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 125         for _, filePath := range []string{"/foo", "/Foo", "/foo/bar"} {
 126                 req := newRequest("GET", collURL+filePath)
 127                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 128                 if !c.Check(actual, check.NotNil) {
 129                         continue
 130                 }
 131                 c.Check(actual.collFilePath, check.Equals, filePath)
 132         }
 133 }
 134
 135 func (s *UnitSuite) TestLogRemoteAddr(c *check.C) {
 136         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 137         filePath := "/foo"
 138         req := newRequest("GET", collURL+filePath)
 139
 140         for _, addr := range []string{"10.20.30.55", "192.168.144.120", "192.0.2.4"} {
 141                 req.RemoteAddr = addr + ":57914"
 142                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 143                 if !c.Check(actual, check.NotNil) {
 144                         continue
 145                 }
 146                 c.Check(actual.clientAddr, check.Equals, addr)
 147         }
 148
 149         for _, addr := range []string{"100::20:30:40", "2001:db8::90:100", "3fff::30"} {
 150                 req.RemoteAddr = fmt.Sprintf("[%s]:57916", addr)
 151                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 152                 if !c.Check(actual, check.NotNil) {
 153                         continue
 154                 }
 155                 c.Check(actual.clientAddr, check.Equals, addr)
 156         }
 157 }
 158
 159 func (s *UnitSuite) TestLogXForwardedFor(c *check.C) {
 160         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 161         filePath := "/foo"
 162         req := newRequest("GET", collURL+filePath)
 163         for xff, expected := range map[string]string{
 164                 "10.20.30.55":                          "10.20.30.55",
 165                 "192.168.144.120, 10.20.30.120":        "10.20.30.120",
 166                 "192.0.2.4, 192.0.2.6, 192.0.2.8":      "192.0.2.8",
 167                 "192.0.2.4,192.168.2.4":                "192.168.2.4",
 168                 "10.20.30.60,192.168.144.40,192.0.2.4": "192.0.2.4",
 169                 "100::20:30:50":                        "100::20:30:50",
 170                 "2001:db8::80:90, 100::100":            "100::100",
 171                 "3fff::ff, 3fff::ee, 3fff::fe":         "3fff::fe",
 172                 "3fff::3f,100::1000":                   "100::1000",
 173                 "2001:db8::88,100::88,3fff::88":        "3fff::88",
 174                 "10.20.30.60, 2001:db8::60":            "2001:db8::60",
 175                 "2001:db8::20,10.20.30.20":             "10.20.30.20",
 176                 ", 10.20.30.123, 100::123":             "100::123",
 177                 ",100::321,10.30.20.10":                "10.30.20.10",
 178         } {
 179                 req.Header.Set("X-Forwarded-For", xff)
 180                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 181                 if !c.Check(actual, check.NotNil) {
 182                         continue
 183                 }
 184                 c.Check(actual.clientAddr, check.Equals, expected)
 185         }
 186 }
 187
 188 func (s *UnitSuite) TestLogXForwardedForMalformed(c *check.C) {
 189         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 190         filePath := "/foo"
 191         req := newRequest("GET", collURL+filePath)
 192         for _, xff := range []string{"", ",", "10.20,30.40", "foo, bar"} {
 193                 req.Header.Set("X-Forwarded-For", xff)
 194                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 195                 if !c.Check(actual, check.NotNil) {
 196                         continue
 197                 }
 198                 c.Check(actual.clientAddr, check.Equals, "10.20.30.40")
 199         }
 200 }
 201
 202 func (s *UnitSuite) TestLogXForwardedForMultivalue(c *check.C) {
 203         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 204         filePath := "/foo"
 205         req := newRequest("GET", collURL+filePath)
 206         req.Header.Set("X-Forwarded-For", ", ")
 207         req.Header.Add("X-Forwarded-For", "2001:db8::db9:dbd")
 208         req.Header.Add("X-Forwarded-For", "10.20.30.90")
 209         actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 210         c.Assert(actual, check.NotNil)
 211         c.Check(actual.clientAddr, check.Equals, "10.20.30.90")
 212 }
 213
 214 func (s *UnitSuite) TestLogClientAddressCanonicalization(c *check.C) {
 215         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 216         filePath := "/foo"
 217         req := newRequest("GET", collURL+filePath)
 218         expected := "2001:db8::12:0"
 219
 220         req.RemoteAddr = "[2001:db8::012:0000]:57918"
 221         a := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 222         c.Assert(a, check.NotNil)
 223         c.Check(a.clientAddr, check.Equals, expected)
 224
 225         req.RemoteAddr = "10.20.30.40:57919"
 226         req.Header.Set("X-Forwarded-For", "2001:db8:0::0:12:00")
 227         b := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 228         c.Assert(b, check.NotNil)
 229         c.Check(b.clientAddr, check.Equals, expected)
 230 }
 231
 232 func (s *UnitSuite) TestLogAnonymousUser(c *check.C) {
 233         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 234         filePath := "/foo"
 235         req := newRequest("GET", collURL+filePath)
 236         actual := newFileEventLog(s.handler, req, filePath, nil, nil, arvadostest.AnonymousToken)
 237         c.Assert(actual, check.NotNil)
 238         c.Check(actual.userUUID, check.Equals, s.handler.Cluster.ClusterID+"-tpzed-anonymouspublic")
 239         c.Check(actual.userFullName, check.Equals, "")
 240         c.Check(actual.clientToken, check.Equals, arvadostest.AnonymousToken)
 241 }
 242
 243 func (s *UnitSuite) TestLogUser(c *check.C) {
 244         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 245         for _, trial := range []struct{ uuid, fullName, token string }{
 246                 {arvadostest.ActiveUserUUID, "Active User", arvadostest.ActiveToken},
 247                 {arvadostest.SpectatorUserUUID, "Spectator User", arvadostest.SpectatorToken},
 248         } {
 249                 filePath := "/" + trial.uuid
 250                 req := newRequest("GET", collURL+filePath)
 251                 user := &arvados.User{
 252                         UUID:     trial.uuid,
 253                         FullName: trial.fullName,
 254                 }
 255                 actual := newFileEventLog(s.handler, req, filePath, nil, user, trial.token)
 256                 if !c.Check(actual, check.NotNil) {
 257                         continue
 258                 }
 259                 c.Check(actual.userUUID, check.Equals, trial.uuid)
 260                 c.Check(actual.userFullName, check.Equals, trial.fullName)
 261                 c.Check(actual.clientToken, check.Equals, trial.token)
 262         }
 263 }
 264
 265 func (s *UnitSuite) TestLogCollectionByUUID(c *check.C) {
 266         for collUUID, collPDH := range arvadostest.TestCollectionUUIDToPDH {
 267                 collURL := "http://keep-web.example/c=" + collUUID
 268                 filePath := "/" + collUUID
 269                 req := newRequest("GET", collURL+filePath)
 270                 coll := newCollection(collUUID)
 271                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 272                 if !c.Check(actual, check.NotNil) {
 273                         continue
 274                 }
 275                 c.Check(actual.collUUID, check.Equals, collUUID)
 276                 c.Check(actual.collPDH, check.Equals, collPDH)
 277         }
 278 }
 279
 280 func (s *UnitSuite) TestLogCollectionByPDH(c *check.C) {
 281         for _, collPDH := range arvadostest.TestCollectionUUIDToPDH {
 282                 collURL := "http://keep-web.example/c=" + collPDH
 283                 filePath := "/PDHFile"
 284                 req := newRequest("GET", collURL+filePath)
 285                 coll := newCollection(collPDH)
 286                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 287                 if !c.Check(actual, check.NotNil) {
 288                         continue
 289                 }
 290                 c.Check(actual.collPDH, check.Equals, collPDH)
 291                 c.Check(actual.collUUID, check.Equals, "")
 292         }
 293 }
 294
 295 func (s *UnitSuite) TestLogGETUUIDAsDict(c *check.C) {
 296         filePath := "/foo"
 297         reqPath := "/c=" + arvadostest.FooCollection + filePath
 298         req := newRequest("GET", "http://keep-web.example"+reqPath)
 299         coll := newCollection(arvadostest.FooCollection)
 300         logEvent := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 301         c.Assert(logEvent, check.NotNil)
 302         c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 303                 "event_type":  "file_download",
 304                 "object_uuid": s.handler.Cluster.ClusterID + "-tpzed-anonymouspublic",
 305                 "properties": arvadosclient.Dict{
 306                         "reqPath":              reqPath,
 307                         "collection_uuid":      arvadostest.FooCollection,
 308                         "collection_file_path": filePath,
 309                         "portable_data_hash":   arvadostest.FooCollectionPDH,
 310                 },
 311         })
 312 }
 313
 314 func (s *UnitSuite) TestLogGETPDHAsDict(c *check.C) {
 315         filePath := "/Foo"
 316         reqPath := "/c=" + arvadostest.FooCollectionPDH + filePath
 317         req := newRequest("GET", "http://keep-web.example"+reqPath)
 318         coll := newCollection(arvadostest.FooCollectionPDH)
 319         user := &arvados.User{
 320                 UUID:     arvadostest.ActiveUserUUID,
 321                 FullName: "Active User",
 322         }
 323         logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 324         c.Assert(logEvent, check.NotNil)
 325         c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 326                 "event_type":  "file_download",
 327                 "object_uuid": arvadostest.ActiveUserUUID,
 328                 "properties": arvadosclient.Dict{
 329                         "reqPath":              reqPath,
 330                         "portable_data_hash":   arvadostest.FooCollectionPDH,
 331                         "collection_uuid":      "",
 332                         "collection_file_path": filePath,
 333                 },
 334         })
 335 }
 336
 337 func (s *UnitSuite) TestLogUploadAsDict(c *check.C) {
 338         coll := newCollection(arvadostest.FooCollection)
 339         user := &arvados.User{
 340                 UUID:     arvadostest.ActiveUserUUID,
 341                 FullName: "Active User",
 342         }
 343         for _, method := range []string{"POST", "PUT"} {
 344                 filePath := "/" + method + "File"
 345                 reqPath := "/c=" + arvadostest.FooCollection + filePath
 346                 req := newRequest(method, "http://keep-web.example"+reqPath)
 347                 logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 348                 if !c.Check(logEvent, check.NotNil) {
 349                         continue
 350                 }
 351                 c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 352                         "event_type":  "file_upload",
 353                         "object_uuid": arvadostest.ActiveUserUUID,
 354                         "properties": arvadosclient.Dict{
 355                                 "reqPath":              reqPath,
 356                                 "collection_uuid":      arvadostest.FooCollection,
 357                                 "collection_file_path": filePath,
 358                         },
 359                 })
 360         }
 361 }
 362
 363 func (s *UnitSuite) TestLogGETUUIDAsFields(c *check.C) {
 364         filePath := "/foo"
 365         reqPath := "/c=" + arvadostest.FooCollection + filePath
 366         req := newRequest("GET", "http://keep-web.example"+reqPath)
 367         coll := newCollection(arvadostest.FooCollection)
 368         logEvent := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 369         c.Assert(logEvent, check.NotNil)
 370         c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 371                 "user_uuid":            s.handler.Cluster.ClusterID + "-tpzed-anonymouspublic",
 372                 "collection_uuid":      arvadostest.FooCollection,
 373                 "collection_file_path": filePath,
 374                 "portable_data_hash":   arvadostest.FooCollectionPDH,
 375         })
 376 }
 377
 378 func (s *UnitSuite) TestLogGETPDHAsFields(c *check.C) {
 379         filePath := "/Foo"
 380         reqPath := "/c=" + arvadostest.FooCollectionPDH + filePath
 381         req := newRequest("GET", "http://keep-web.example"+reqPath)
 382         coll := newCollection(arvadostest.FooCollectionPDH)
 383         user := &arvados.User{
 384                 UUID:     arvadostest.ActiveUserUUID,
 385                 FullName: "Active User",
 386         }
 387         logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 388         c.Assert(logEvent, check.NotNil)
 389         c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 390                 "user_uuid":            arvadostest.ActiveUserUUID,
 391                 "user_full_name":       "Active User",
 392                 "collection_uuid":      "",
 393                 "collection_file_path": filePath,
 394                 "portable_data_hash":   arvadostest.FooCollectionPDH,
 395         })
 396 }
 397
 398 func (s *UnitSuite) TestLogUploadAsFields(c *check.C) {
 399         coll := newCollection(arvadostest.FooCollection)
 400         user := &arvados.User{
 401                 UUID:     arvadostest.ActiveUserUUID,
 402                 FullName: "Active User",
 403         }
 404         for _, method := range []string{"POST", "PUT"} {
 405                 filePath := "/" + method + "File"
 406                 reqPath := "/c=" + arvadostest.FooCollection + filePath
 407                 req := newRequest(method, "http://keep-web.example"+reqPath)
 408                 logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 409                 if !c.Check(logEvent, check.NotNil) {
 410                         continue
 411                 }
 412                 c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 413                         "user_uuid":            arvadostest.ActiveUserUUID,
 414                         "user_full_name":       "Active User",
 415                         "collection_uuid":      arvadostest.FooCollection,
 416                         "collection_file_path": filePath,
 417                 })
 418         }
 419 }
 420
 421 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
 422         h := s.handler
 423         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
 424         req := &http.Request{
 425                 Method:     "OPTIONS",
 426                 Host:       u.Host,
 427                 URL:        u,
 428                 RequestURI: u.RequestURI(),
 429                 Header: http.Header{
 430                         "Origin":                        {"https://workbench.example"},
 431                         "Access-Control-Request-Method": {"POST"},
 432                 },
 433         }
 434
 435         // Check preflight for an allowed request
 436         resp := httptest.NewRecorder()
 437         h.ServeHTTP(resp, req)
 438         c.Check(resp.Code, check.Equals, http.StatusOK)
 439         c.Check(resp.Body.String(), check.Equals, "")
 440         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 441         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
 442         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
 443
 444         // Check preflight for a disallowed request
 445         resp = httptest.NewRecorder()
 446         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
 447         h.ServeHTTP(resp, req)
 448         c.Check(resp.Body.String(), check.Equals, "")
 449         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
 450 }
 451
 452 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
 453         for _, trial := range []struct {
 454                 method   string
 455                 path     string
 456                 prefix   string
 457                 source   string
 458                 notFound bool
 459                 seeOther bool
 460         }{
 461                 {
 462                         method: "PROPFIND",
 463                         path:   "/",
 464                 },
 465                 {
 466                         method: "PROPFIND",
 467                         path:   "/dir1",
 468                 },
 469                 {
 470                         method: "PROPFIND",
 471                         path:   "/dir1/",
 472                 },
 473                 {
 474                         method: "PROPFIND",
 475                         path:   "/dir1/foo",
 476                         prefix: "/dir1",
 477                         source: "/dir1",
 478                 },
 479                 {
 480                         method: "PROPFIND",
 481                         path:   "/prefix/dir1/foo",
 482                         prefix: "/prefix/",
 483                         source: "",
 484                 },
 485                 {
 486                         method: "PROPFIND",
 487                         path:   "/prefix/dir1/foo",
 488                         prefix: "/prefix",
 489                         source: "",
 490                 },
 491                 {
 492                         method: "PROPFIND",
 493                         path:   "/prefix/dir1/foo",
 494                         prefix: "/prefix/",
 495                         source: "/",
 496                 },
 497                 {
 498                         method: "PROPFIND",
 499                         path:   "/prefix/foo",
 500                         prefix: "/prefix/",
 501                         source: "/dir1/",
 502                 },
 503                 {
 504                         method: "GET",
 505                         path:   "/prefix/foo",
 506                         prefix: "/prefix/",
 507                         source: "/dir1/",
 508                 },
 509                 {
 510                         method: "PROPFIND",
 511                         path:   "/prefix/",
 512                         prefix: "/prefix",
 513                         source: "/dir1",
 514                 },
 515                 {
 516                         method: "PROPFIND",
 517                         path:   "/prefix",
 518                         prefix: "/prefix",
 519                         source: "/dir1/",
 520                 },
 521                 {
 522                         method:   "GET",
 523                         path:     "/prefix",
 524                         prefix:   "/prefix",
 525                         source:   "/dir1",
 526                         seeOther: true,
 527                 },
 528                 {
 529                         method:   "PROPFIND",
 530                         path:     "/dir1/foo",
 531                         prefix:   "",
 532                         source:   "/dir1",
 533                         notFound: true,
 534                 },
 535         } {
 536                 c.Logf("trial %+v", trial)
 537                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 538                 req := &http.Request{
 539                         Method:     trial.method,
 540                         Host:       u.Host,
 541                         URL:        u,
 542                         RequestURI: u.RequestURI(),
 543                         Header: http.Header{
 544                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 545                                 "X-Webdav-Prefix": {trial.prefix},
 546                                 "X-Webdav-Source": {trial.source},
 547                         },
 548                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 549                 }
 550
 551                 resp := httptest.NewRecorder()
 552                 s.handler.ServeHTTP(resp, req)
 553                 if trial.notFound {
 554                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 555                 } else if trial.method == "PROPFIND" {
 556                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 557                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 558                 } else if trial.seeOther {
 559                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 560                 } else {
 561                         c.Check(resp.Code, check.Equals, http.StatusOK)
 562                 }
 563         }
 564 }
 565
 566 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 567         // Ensure we start with an empty cache
 568         defer os.Setenv("HOME", os.Getenv("HOME"))
 569         os.Setenv("HOME", c.MkDir())
 570         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(0)
 571
 572         for _, trial := range []struct {
 573                 dataExists    bool
 574                 sendIMSHeader bool
 575                 expectStatus  int
 576                 logRegexp     string
 577         }{
 578                 // If we return no content due to a Keep read error,
 579                 // we should emit a log message.
 580                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 581
 582                 // If we return no content because the client sent an
 583                 // If-Modified-Since header, our response should be
 584                 // 304.  We still expect a "File download" log since it
 585                 // counts as a file access for auditing.
 586                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 587         } {
 588                 c.Logf("trial: %+v", trial)
 589                 arvadostest.StartKeep(2, true)
 590                 if trial.dataExists {
 591                         arv, err := arvadosclient.MakeArvadosClient()
 592                         c.Assert(err, check.IsNil)
 593                         arv.ApiToken = arvadostest.ActiveToken
 594                         kc, err := keepclient.MakeKeepClient(arv)
 595                         c.Assert(err, check.IsNil)
 596                         _, _, err = kc.PutB([]byte("foo"))
 597                         c.Assert(err, check.IsNil)
 598                 }
 599
 600                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 601                 req := &http.Request{
 602                         Method:     "GET",
 603                         Host:       u.Host,
 604                         URL:        u,
 605                         RequestURI: u.RequestURI(),
 606                         Header: http.Header{
 607                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 608                         },
 609                 }
 610                 if trial.sendIMSHeader {
 611                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 612                 }
 613
 614                 var logbuf bytes.Buffer
 615                 logger := logrus.New()
 616                 logger.Out = &logbuf
 617                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 618
 619                 resp := httptest.NewRecorder()
 620                 s.handler.ServeHTTP(resp, req)
 621                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 622                 c.Check(resp.Body.String(), check.Equals, "")
 623
 624                 c.Log(logbuf.String())
 625                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 626         }
 627 }
 628
 629 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 630         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 631         token := arvadostest.ActiveToken
 632         for _, trial := range []string{
 633                 "http://keep-web/c=" + bogusID + "/foo",
 634                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 635                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 636                 "http://keep-web/collections/" + bogusID + "/foo",
 637                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 638                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 639         } {
 640                 c.Log(trial)
 641                 u := mustParseURL(trial)
 642                 req := &http.Request{
 643                         Method:     "GET",
 644                         Host:       u.Host,
 645                         URL:        u,
 646                         RequestURI: u.RequestURI(),
 647                 }
 648                 resp := httptest.NewRecorder()
 649                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 650                 s.handler.ServeHTTP(resp, req)
 651                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 652         }
 653 }
 654
 655 func mustParseURL(s string) *url.URL {
 656         r, err := url.Parse(s)
 657         if err != nil {
 658                 panic("parse URL: " + s)
 659         }
 660         return r
 661 }
 662
 663 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 664         for _, testURL := range []string{
 665                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 666                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 667         } {
 668                 resp := httptest.NewRecorder()
 669                 u := mustParseURL(testURL)
 670                 req := &http.Request{
 671                         Method:     "GET",
 672                         URL:        u,
 673                         RequestURI: u.RequestURI(),
 674                 }
 675                 s.handler.ServeHTTP(resp, req)
 676                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 677                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 678         }
 679 }
 680
 681 // An authorizer modifies an HTTP request to make use of the given
 682 // token -- by adding it to a header, cookie, query param, or whatever
 683 // -- and returns the HTTP status code we should expect from keep-web if
 684 // the token is invalid.
 685 type authorizer func(*http.Request, string) int
 686
 687 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 688         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 689 }
 690 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 691         r.Header.Add("Authorization", "OAuth2 "+tok)
 692         return http.StatusUnauthorized
 693 }
 694
 695 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 696         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 697 }
 698 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 699         r.Header.Add("Authorization", "Bearer "+tok)
 700         return http.StatusUnauthorized
 701 }
 702
 703 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 704         s.doVhostRequests(c, authzViaCookieValue)
 705 }
 706 func authzViaCookieValue(r *http.Request, tok string) int {
 707         r.AddCookie(&http.Cookie{
 708                 Name:  "arvados_api_token",
 709                 Value: auth.EncodeTokenCookie([]byte(tok)),
 710         })
 711         return http.StatusUnauthorized
 712 }
 713
 714 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 715         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 716 }
 717 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 718         r.AddCookie(&http.Cookie{
 719                 Name:  "arvados_api_token",
 720                 Value: auth.EncodeTokenCookie([]byte(tok)),
 721         })
 722         return http.StatusUnauthorized
 723 }
 724
 725 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 726         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 727                 r.AddCookie(&http.Cookie{
 728                         Name:  "arvados_api_token",
 729                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 730                 })
 731                 return http.StatusUnauthorized
 732         })
 733 }
 734
 735 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 736         s.doVhostRequests(c, authzViaPath)
 737 }
 738 func authzViaPath(r *http.Request, tok string) int {
 739         r.URL.Path = "/t=" + tok + r.URL.Path
 740         return http.StatusNotFound
 741 }
 742
 743 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 744         s.doVhostRequests(c, authzViaQueryString)
 745 }
 746 func authzViaQueryString(r *http.Request, tok string) int {
 747         r.URL.RawQuery = "api_token=" + tok
 748         return http.StatusUnauthorized
 749 }
 750
 751 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 752         s.doVhostRequests(c, authzViaPOST)
 753 }
 754 func authzViaPOST(r *http.Request, tok string) int {
 755         r.Method = "POST"
 756         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 757         r.Body = ioutil.NopCloser(strings.NewReader(
 758                 url.Values{"api_token": {tok}}.Encode()))
 759         return http.StatusUnauthorized
 760 }
 761
 762 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 763         s.doVhostRequests(c, authzViaPOST)
 764 }
 765 func authzViaXHRPOST(r *http.Request, tok string) int {
 766         r.Method = "POST"
 767         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 768         r.Header.Add("Origin", "https://origin.example")
 769         r.Body = ioutil.NopCloser(strings.NewReader(
 770                 url.Values{
 771                         "api_token":   {tok},
 772                         "disposition": {"attachment"},
 773                 }.Encode()))
 774         return http.StatusUnauthorized
 775 }
 776
 777 // Try some combinations of {url, token} using the given authorization
 778 // mechanism, and verify the result is correct.
 779 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 780         for _, hostPath := range []string{
 781                 arvadostest.FooCollection + ".example.com/foo",
 782                 arvadostest.FooCollection + "--collections.example.com/foo",
 783                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 784                 arvadostest.FooCollectionPDH + ".example.com/foo",
 785                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 786                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 787         } {
 788                 c.Log("doRequests: ", hostPath)
 789                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 790         }
 791 }
 792
 793 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 794         for _, tok := range []string{
 795                 arvadostest.ActiveToken,
 796                 arvadostest.ActiveToken[:15],
 797                 arvadostest.SpectatorToken,
 798                 "bogus",
 799                 "",
 800         } {
 801                 u := mustParseURL("http://" + hostPath)
 802                 req := &http.Request{
 803                         Method:     "GET",
 804                         Host:       u.Host,
 805                         URL:        u,
 806                         RequestURI: u.RequestURI(),
 807                         Header:     http.Header{},
 808                 }
 809                 failCode := authz(req, tok)
 810                 req, resp := s.doReq(req)
 811                 code, body := resp.Code, resp.Body.String()
 812
 813                 // If the initial request had a (non-empty) token
 814                 // showing in the query string, we should have been
 815                 // redirected in order to hide it in a cookie.
 816                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 817
 818                 if tok == arvadostest.ActiveToken {
 819                         c.Check(code, check.Equals, http.StatusOK)
 820                         c.Check(body, check.Equals, "foo")
 821                 } else {
 822                         c.Check(code >= 400, check.Equals, true)
 823                         c.Check(code < 500, check.Equals, true)
 824                         if tok == arvadostest.SpectatorToken {
 825                                 // Valid token never offers to retry
 826                                 // with different credentials.
 827                                 c.Check(code, check.Equals, http.StatusNotFound)
 828                         } else {
 829                                 // Invalid token can ask to retry
 830                                 // depending on the authz method.
 831                                 c.Check(code, check.Equals, failCode)
 832                         }
 833                         if code == 404 {
 834                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 835                         } else {
 836                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 837                         }
 838                 }
 839         }
 840 }
 841
 842 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 843         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 844                 for _, port := range []string{"80", "443", "8000"} {
 845                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 846                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 847                         req := &http.Request{
 848                                 Method:     "GET",
 849                                 Host:       u.Host,
 850                                 URL:        u,
 851                                 RequestURI: u.RequestURI(),
 852                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 853                         }
 854                         req, resp := s.doReq(req)
 855                         code, _ := resp.Code, resp.Body.String()
 856
 857                         if port == "8000" {
 858                                 c.Check(code, check.Equals, 401)
 859                         } else {
 860                                 c.Check(code, check.Equals, 200)
 861                         }
 862                 }
 863         }
 864 }
 865
 866 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 867         u := mustParseURL(urlstring)
 868         if hdr == nil && token != "" {
 869                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 870         } else if hdr == nil {
 871                 hdr = http.Header{}
 872         } else if token != "" {
 873                 panic("must not pass both token and hdr")
 874         }
 875         return s.doReq(&http.Request{
 876                 Method:     method,
 877                 Host:       u.Host,
 878                 URL:        u,
 879                 RequestURI: u.RequestURI(),
 880                 Header:     hdr,
 881         })
 882 }
 883
 884 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 885         resp := httptest.NewRecorder()
 886         s.handler.ServeHTTP(resp, req)
 887         if resp.Code != http.StatusSeeOther {
 888                 return req, resp
 889         }
 890         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 891         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 892         req = &http.Request{
 893                 Method:     "GET",
 894                 Host:       u.Host,
 895                 URL:        u,
 896                 RequestURI: u.RequestURI(),
 897                 Header:     http.Header{},
 898         }
 899         for _, c := range cookies {
 900                 req.AddCookie(c)
 901         }
 902         return s.doReq(req)
 903 }
 904
 905 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 906         s.testVhostRedirectTokenToCookie(c, "GET",
 907                 arvadostest.FooCollection+".example.com/foo",
 908                 "?api_token="+arvadostest.ActiveToken,
 909                 nil,
 910                 "",
 911                 http.StatusOK,
 912                 "foo",
 913         )
 914 }
 915
 916 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 917         s.testVhostRedirectTokenToCookie(c, "GET",
 918                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 919                 "",
 920                 nil,
 921                 "",
 922                 http.StatusOK,
 923                 "foo",
 924         )
 925 }
 926
 927 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 928         s.testVhostRedirectTokenToCookie(c, "GET",
 929                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 930                 "",
 931                 nil,
 932                 "",
 933                 http.StatusOK,
 934                 "foo",
 935         )
 936         // Same valid sharing token, but requesting a different collection
 937         s.testVhostRedirectTokenToCookie(c, "GET",
 938                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 939                 "",
 940                 nil,
 941                 "",
 942                 http.StatusNotFound,
 943                 regexp.QuoteMeta(notFoundMessage+"\n"),
 944         )
 945 }
 946
 947 // Bad token in URL is 404 Not Found because it doesn't make sense to
 948 // retry the same URL with different authorization.
 949 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 950         s.testVhostRedirectTokenToCookie(c, "GET",
 951                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 952                 "",
 953                 nil,
 954                 "",
 955                 http.StatusNotFound,
 956                 regexp.QuoteMeta(notFoundMessage+"\n"),
 957         )
 958 }
 959
 960 // Bad token in a cookie (even if it got there via our own
 961 // query-string-to-cookie redirect) is, in principle, retryable via
 962 // wb2-login-and-redirect flow.
 963 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 964         // Inline
 965         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 966                 arvadostest.FooCollection+".example.com/foo",
 967                 "?api_token=thisisabogustoken",
 968                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 969                 "",
 970                 http.StatusSeeOther,
 971                 "",
 972         )
 973         u, err := url.Parse(resp.Header().Get("Location"))
 974         c.Assert(err, check.IsNil)
 975         c.Logf("redirected to %s", u)
 976         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 977         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 978         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 979
 980         // Download/attachment indicated by ?disposition=attachment
 981         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 982                 arvadostest.FooCollection+".example.com/foo",
 983                 "?api_token=thisisabogustoken&disposition=attachment",
 984                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 985                 "",
 986                 http.StatusSeeOther,
 987                 "",
 988         )
 989         u, err = url.Parse(resp.Header().Get("Location"))
 990         c.Assert(err, check.IsNil)
 991         c.Logf("redirected to %s", u)
 992         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 993         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 994         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 995
 996         // Download/attachment indicated by vhost
 997         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 998                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 999                 "?api_token=thisisabogustoken",
1000                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
1001                 "",
1002                 http.StatusSeeOther,
1003                 "",
1004         )
1005         u, err = url.Parse(resp.Header().Get("Location"))
1006         c.Assert(err, check.IsNil)
1007         c.Logf("redirected to %s", u)
1008         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1009         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
1010         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1011
1012         // Without "Sec-Fetch-Mode: navigate" header, just 401.
1013         s.testVhostRedirectTokenToCookie(c, "GET",
1014                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1015                 "?api_token=thisisabogustoken",
1016                 http.Header{"Sec-Fetch-Mode": {"cors"}},
1017                 "",
1018                 http.StatusUnauthorized,
1019                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
1020         )
1021         s.testVhostRedirectTokenToCookie(c, "GET",
1022                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1023                 "?api_token=thisisabogustoken",
1024                 nil,
1025                 "",
1026                 http.StatusUnauthorized,
1027                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
1028         )
1029 }
1030
1031 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
1032         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1033                 arvadostest.FooCollection+".example.com/foo",
1034                 "?api_token=thisisabogustoken",
1035                 http.Header{
1036                         "Sec-Fetch-Mode": {"navigate"},
1037                         "Cache-Control":  {"no-cache"},
1038                 },
1039                 "",
1040                 http.StatusSeeOther,
1041                 "",
1042         )
1043         u, err := url.Parse(resp.Header().Get("Location"))
1044         c.Assert(err, check.IsNil)
1045         c.Logf("redirected to %s", u)
1046         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1047         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1048         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
1049 }
1050
1051 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
1052         for _, trial := range []struct {
1053                 anonToken    bool
1054                 cacheControl string
1055         }{
1056                 {},
1057                 {cacheControl: "no-cache"},
1058                 {anonToken: true},
1059                 {anonToken: true, cacheControl: "no-cache"},
1060         } {
1061                 c.Logf("trial: %+v", trial)
1062
1063                 if trial.anonToken {
1064                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1065                 } else {
1066                         s.handler.Cluster.Users.AnonymousUserToken = ""
1067                 }
1068                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1069                 c.Assert(err, check.IsNil)
1070                 req.Header.Set("Sec-Fetch-Mode", "navigate")
1071                 if trial.cacheControl != "" {
1072                         req.Header.Set("Cache-Control", trial.cacheControl)
1073                 }
1074                 resp := httptest.NewRecorder()
1075                 s.handler.ServeHTTP(resp, req)
1076                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
1077                 u, err := url.Parse(resp.Header().Get("Location"))
1078                 c.Assert(err, check.IsNil)
1079                 c.Logf("redirected to %q", u)
1080                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1081                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1082                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
1083         }
1084 }
1085
1086 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
1087         s.testVhostRedirectTokenToCookie(c, "GET",
1088                 "example.com/c="+arvadostest.FooCollection+"/foo",
1089                 "?api_token="+arvadostest.ActiveToken,
1090                 nil,
1091                 "",
1092                 http.StatusBadRequest,
1093                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
1094         )
1095 }
1096
1097 // If client requests an attachment by putting ?disposition=attachment
1098 // in the query string, and gets redirected, the redirect target
1099 // should respond with an attachment.
1100 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
1101         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1102                 arvadostest.FooCollection+".example.com/foo",
1103                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
1104                 nil,
1105                 "",
1106                 http.StatusOK,
1107                 "foo",
1108         )
1109         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1110 }
1111
1112 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
1113         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1114         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1115                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
1116                 "?api_token="+arvadostest.ActiveToken,
1117                 nil,
1118                 "",
1119                 http.StatusOK,
1120                 "foo",
1121         )
1122         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1123 }
1124
1125 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
1126         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1127         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1128                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
1129                 "?api_token="+arvadostest.ActiveToken,
1130                 nil,
1131                 "",
1132                 http.StatusOK,
1133                 "waz",
1134         )
1135         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1136         resp = s.testVhostRedirectTokenToCookie(c, "GET",
1137                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
1138                 "?api_token="+arvadostest.ActiveToken,
1139                 nil,
1140                 "",
1141                 http.StatusOK,
1142                 "waz",
1143         )
1144         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1145 }
1146
1147 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
1148         s.handler.Cluster.Collections.TrustAllContent = true
1149         s.testVhostRedirectTokenToCookie(c, "GET",
1150                 "example.com/c="+arvadostest.FooCollection+"/foo",
1151                 "?api_token="+arvadostest.ActiveToken,
1152                 nil,
1153                 "",
1154                 http.StatusOK,
1155                 "foo",
1156         )
1157 }
1158
1159 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
1160         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
1161
1162         s.testVhostRedirectTokenToCookie(c, "GET",
1163                 "example.com/c="+arvadostest.FooCollection+"/foo",
1164                 "?api_token="+arvadostest.ActiveToken,
1165                 nil,
1166                 "",
1167                 http.StatusBadRequest,
1168                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
1169         )
1170
1171         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1172                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
1173                 "?api_token="+arvadostest.ActiveToken,
1174                 nil,
1175                 "",
1176                 http.StatusOK,
1177                 "foo",
1178         )
1179         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
1180 }
1181
1182 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
1183         baseUrl := arvadostest.FooCollection + ".example.com/foo"
1184         query := url.Values{}
1185
1186         // The intent of these tests is to check that requests are redirected
1187         // correctly in the presence of multiple API tokens. The exact response
1188         // codes and content are not closely considered: they're just how
1189         // keep-web responded when we made the smallest possible fix. Changing
1190         // those responses may be okay, but you should still test all these
1191         // different cases and the associated redirect logic.
1192         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
1193         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1194         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
1195         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1196         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
1197         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1198         query["api_token"] = []string{"", arvadostest.ActiveToken}
1199         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1200
1201         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
1202         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
1203         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1204         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
1205         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1206         query["api_token"] = []string{"", arvadostest.AnonymousToken}
1207         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1208 }
1209
1210 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
1211         s.testVhostRedirectTokenToCookie(c, "POST",
1212                 arvadostest.FooCollection+".example.com/foo",
1213                 "",
1214                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
1215                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
1216                 http.StatusOK,
1217                 "foo",
1218         )
1219 }
1220
1221 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
1222         s.testVhostRedirectTokenToCookie(c, "POST",
1223                 arvadostest.FooCollection+".example.com/foo",
1224                 "",
1225                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
1226                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
1227                 http.StatusNotFound,
1228                 regexp.QuoteMeta(notFoundMessage+"\n"),
1229         )
1230 }
1231
1232 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
1233         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1234         s.testVhostRedirectTokenToCookie(c, "GET",
1235                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
1236                 "",
1237                 nil,
1238                 "",
1239                 http.StatusOK,
1240                 "Hello world\n",
1241         )
1242 }
1243
1244 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
1245         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
1246         s.testVhostRedirectTokenToCookie(c, "GET",
1247                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
1248                 "",
1249                 nil,
1250                 "",
1251                 http.StatusUnauthorized,
1252                 "Authorization tokens are not accepted here: .*\n",
1253         )
1254 }
1255
1256 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
1257         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1258
1259         client := arvados.NewClientFromEnv()
1260         client.AuthToken = arvadostest.ActiveToken
1261         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
1262         c.Assert(err, check.IsNil)
1263         path := `https:\\"odd' path chars`
1264         f, err := fs.OpenFile(path, os.O_CREATE, 0777)
1265         c.Assert(err, check.IsNil)
1266         f.Close()
1267         mtxt, err := fs.MarshalManifest(".")
1268         c.Assert(err, check.IsNil)
1269         var coll arvados.Collection
1270         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1271                 "collection": map[string]string{
1272                         "manifest_text": mtxt,
1273                 },
1274         })
1275         c.Assert(err, check.IsNil)
1276
1277         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
1278         req := &http.Request{
1279                 Method:     "GET",
1280                 Host:       u.Host,
1281                 URL:        u,
1282                 RequestURI: u.RequestURI(),
1283                 Header: http.Header{
1284                         "Authorization": {"Bearer " + client.AuthToken},
1285                 },
1286         }
1287         resp := httptest.NewRecorder()
1288         s.handler.ServeHTTP(resp, req)
1289         c.Check(resp.Code, check.Equals, http.StatusOK)
1290         doc, err := html.Parse(resp.Body)
1291         c.Assert(err, check.IsNil)
1292         pathHrefMap := getPathHrefMap(doc)
1293         c.Check(pathHrefMap, check.HasLen, 1) // the one leaf added to collection
1294         href, hasPath := pathHrefMap[path]
1295         c.Assert(hasPath, check.Equals, true) // the path is listed
1296         relUrl := mustParseURL(href)
1297         c.Check(relUrl.Path, check.Equals, "./"+path) // href can be decoded back to path
1298 }
1299
1300 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
1301         arv := arvados.NewClientFromEnv()
1302         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1303         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
1304         name := "foo/bar/baz"
1305         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
1306
1307         client := arvados.NewClientFromEnv()
1308         client.AuthToken = arvadostest.ActiveToken
1309         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
1310         c.Assert(err, check.IsNil)
1311         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
1312         c.Assert(err, check.IsNil)
1313         f.Close()
1314         mtxt, err := fs.MarshalManifest(".")
1315         c.Assert(err, check.IsNil)
1316         var coll arvados.Collection
1317         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1318                 "collection": map[string]string{
1319                         "manifest_text": mtxt,
1320                         "name":          name,
1321                         "owner_uuid":    arvadostest.AProjectUUID,
1322                 },
1323         })
1324         c.Assert(err, check.IsNil)
1325         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1326
1327         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
1328         for tryURL, expectedAnchorText := range map[string]string{
1329                 base:                   nameShown + "/",
1330                 base + nameShown + "/": "filename",
1331         } {
1332                 u, _ := url.Parse(tryURL)
1333                 req := &http.Request{
1334                         Method:     "GET",
1335                         Host:       u.Host,
1336                         URL:        u,
1337                         RequestURI: u.RequestURI(),
1338                         Header: http.Header{
1339                                 "Authorization": {"Bearer " + client.AuthToken},
1340                         },
1341                 }
1342                 resp := httptest.NewRecorder()
1343                 s.handler.ServeHTTP(resp, req)
1344                 c.Check(resp.Code, check.Equals, http.StatusOK)
1345                 doc, err := html.Parse(resp.Body)
1346                 c.Assert(err, check.IsNil) // valid HTML
1347                 pathHrefMap := getPathHrefMap(doc)
1348                 href, hasExpected := pathHrefMap[expectedAnchorText]
1349                 c.Assert(hasExpected, check.Equals, true) // has expected anchor text
1350                 c.Assert(href, check.Not(check.Equals), "")
1351                 relUrl := mustParseURL(href)
1352                 c.Check(relUrl.Path, check.Equals, "./"+expectedAnchorText) // decoded href maps back to the anchor text
1353         }
1354 }
1355
1356 // XHRs can't follow redirect-with-cookie so they rely on method=POST
1357 // and disposition=attachment (telling us it's acceptable to respond
1358 // with content instead of a redirect) and an Origin header that gets
1359 // added automatically by the browser (telling us it's desirable to do
1360 // so).
1361 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
1362         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
1363         req := &http.Request{
1364                 Method:     "POST",
1365                 Host:       u.Host,
1366                 URL:        u,
1367                 RequestURI: u.RequestURI(),
1368                 Header: http.Header{
1369                         "Origin":       {"https://origin.example"},
1370                         "Content-Type": {"application/x-www-form-urlencoded"},
1371                 },
1372                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1373                         "api_token":   {arvadostest.ActiveToken},
1374                         "disposition": {"attachment"},
1375                 }.Encode())),
1376         }
1377         resp := httptest.NewRecorder()
1378         s.handler.ServeHTTP(resp, req)
1379         c.Check(resp.Code, check.Equals, http.StatusOK)
1380         c.Check(resp.Body.String(), check.Equals, "foo")
1381         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1382
1383         // GET + Origin header is representative of both AJAX GET
1384         // requests and inline images via <IMG crossorigin="anonymous"
1385         // src="...">.
1386         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1387         req = &http.Request{
1388                 Method:     "GET",
1389                 Host:       u.Host,
1390                 URL:        u,
1391                 RequestURI: u.RequestURI(),
1392                 Header: http.Header{
1393                         "Origin": {"https://origin.example"},
1394                 },
1395         }
1396         resp = httptest.NewRecorder()
1397         s.handler.ServeHTTP(resp, req)
1398         c.Check(resp.Code, check.Equals, http.StatusOK)
1399         c.Check(resp.Body.String(), check.Equals, "foo")
1400         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1401 }
1402
1403 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1404         if reqHeader == nil {
1405                 reqHeader = http.Header{}
1406         }
1407         u, _ := url.Parse(`http://` + hostPath + queryString)
1408         c.Logf("requesting %s", u)
1409         req := &http.Request{
1410                 Method:     method,
1411                 Host:       u.Host,
1412                 URL:        u,
1413                 RequestURI: u.RequestURI(),
1414                 Header:     reqHeader,
1415                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1416         }
1417
1418         resp := httptest.NewRecorder()
1419         defer func() {
1420                 c.Check(resp.Code, check.Equals, expectStatus)
1421                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1422         }()
1423
1424         s.handler.ServeHTTP(resp, req)
1425         if resp.Code != http.StatusSeeOther {
1426                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1427                 // Since we're not redirecting, check that any api_token in the URL is
1428                 // handled safely.
1429                 // If there is no token in the URL, then we're good.
1430                 // Otherwise, if the response code is an error, the body is expected to
1431                 // be static content, and nothing that might maliciously introspect the
1432                 // URL. It's considered safe and allowed.
1433                 // Otherwise, if the response content has attachment disposition,
1434                 // that's considered safe for all the reasons explained in the
1435                 // safeAttachment comment in handler.go.
1436                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1437                 return resp
1438         }
1439
1440         loc, err := url.Parse(resp.Header().Get("Location"))
1441         c.Assert(err, check.IsNil)
1442         c.Check(loc.Scheme, check.Equals, u.Scheme)
1443         c.Check(loc.Host, check.Equals, u.Host)
1444         c.Check(loc.RawPath, check.Equals, u.RawPath)
1445         // If the response was a redirect, it should never include an API token.
1446         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1447         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1448         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1449
1450         c.Logf("following redirect to %s", u)
1451         req = &http.Request{
1452                 Method:     "GET",
1453                 Host:       loc.Host,
1454                 URL:        loc,
1455                 RequestURI: loc.RequestURI(),
1456                 Header:     reqHeader,
1457         }
1458         for _, c := range cookies {
1459                 req.AddCookie(c)
1460         }
1461
1462         resp = httptest.NewRecorder()
1463         s.handler.ServeHTTP(resp, req)
1464
1465         if resp.Code != http.StatusSeeOther {
1466                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1467         }
1468         return resp
1469 }
1470
1471 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1472         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1473         s.testDirectoryListing(c)
1474 }
1475
1476 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1477         s.handler.Cluster.Users.AnonymousUserToken = ""
1478         s.testDirectoryListing(c)
1479 }
1480
1481 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1482         // The "ownership cycle" test fixtures are reachable from the
1483         // "filter group without filters" group, causing webdav's
1484         // walkfs to recurse indefinitely. Avoid that by deleting one
1485         // of the bogus fixtures.
1486         arv := arvados.NewClientFromEnv()
1487         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1488         if err != nil {
1489                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1490                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1491         }
1492
1493         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1494         authHeader := http.Header{
1495                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1496         }
1497         for _, trial := range []struct {
1498                 uri      string
1499                 header   http.Header
1500                 expect   []string
1501                 redirect string
1502                 cutDirs  int
1503         }{
1504                 {
1505                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1506                         header:  authHeader,
1507                         expect:  []string{"dir1/foo", "dir1/bar"},
1508                         cutDirs: 0,
1509                 },
1510                 {
1511                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1512                         header:  authHeader,
1513                         expect:  []string{"foo", "bar"},
1514                         cutDirs: 1,
1515                 },
1516                 {
1517                         // URLs of this form ignore authHeader, and
1518                         // FooAndBarFilesInDirUUID isn't public, so
1519                         // this returns 401.
1520                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1521                         header: authHeader,
1522                         expect: nil,
1523                 },
1524                 {
1525                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1526                         header:  authHeader,
1527                         expect:  []string{"dir1/"},
1528                         cutDirs: 3,
1529                 },
1530                 {
1531                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1532                         header:  authHeader,
1533                         expect:  []string{"bar"},
1534                         cutDirs: 4,
1535                 },
1536                 {
1537                         uri:     "download.example.com/",
1538                         header:  authHeader,
1539                         expect:  []string{"users/"},
1540                         cutDirs: 0,
1541                 },
1542                 {
1543                         uri:      "download.example.com/users",
1544                         header:   authHeader,
1545                         redirect: "/users/",
1546                         expect:   []string{"active/"},
1547                         cutDirs:  1,
1548                 },
1549                 {
1550                         uri:     "download.example.com/users/",
1551                         header:  authHeader,
1552                         expect:  []string{"active/"},
1553                         cutDirs: 1,
1554                 },
1555                 {
1556                         uri:      "download.example.com/users/active",
1557                         header:   authHeader,
1558                         redirect: "/users/active/",
1559                         expect:   []string{"foo_file_in_dir/"},
1560                         cutDirs:  2,
1561                 },
1562                 {
1563                         uri:     "download.example.com/users/active/",
1564                         header:  authHeader,
1565                         expect:  []string{"foo_file_in_dir/"},
1566                         cutDirs: 2,
1567                 },
1568                 {
1569                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1570                         header:  nil,
1571                         expect:  []string{"dir1/foo", "dir1/bar"},
1572                         cutDirs: 4,
1573                 },
1574                 {
1575                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1576                         header:  nil,
1577                         expect:  []string{"dir1/foo", "dir1/bar"},
1578                         cutDirs: 2,
1579                 },
1580                 {
1581                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1582                         header:  nil,
1583                         expect:  []string{"dir1/foo", "dir1/bar"},
1584                         cutDirs: 2,
1585                 },
1586                 {
1587                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1588                         header:  authHeader,
1589                         expect:  []string{"dir1/foo", "dir1/bar"},
1590                         cutDirs: 1,
1591                 },
1592                 {
1593                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1594                         header:   authHeader,
1595                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1596                         expect:   []string{"foo", "bar"},
1597                         cutDirs:  2,
1598                 },
1599                 {
1600                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1601                         header:  authHeader,
1602                         expect:  []string{"foo", "bar"},
1603                         cutDirs: 3,
1604                 },
1605                 {
1606                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1607                         header:   authHeader,
1608                         redirect: "/dir1/",
1609                         expect:   []string{"foo", "bar"},
1610                         cutDirs:  1,
1611                 },
1612                 {
1613                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1614                         header: authHeader,
1615                         expect: nil,
1616                 },
1617                 {
1618                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1619                         header:  authHeader,
1620                         expect:  []string{"waz"},
1621                         cutDirs: 1,
1622                 },
1623                 {
1624                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1625                         header:  authHeader,
1626                         expect:  []string{"waz"},
1627                         cutDirs: 2,
1628                 },
1629                 {
1630                         uri:     "download.example.com/users/active/This filter group/",
1631                         header:  authHeader,
1632                         expect:  []string{"A Subproject/"},
1633                         cutDirs: 3,
1634                 },
1635                 {
1636                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1637                         header:  authHeader,
1638                         expect:  []string{"baz_file/"},
1639                         cutDirs: 4,
1640                 },
1641                 {
1642                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1643                         header:  authHeader,
1644                         expect:  []string{"A Subproject/"},
1645                         cutDirs: 2,
1646                 },
1647                 {
1648                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1649                         header:  authHeader,
1650                         expect:  []string{"baz_file/"},
1651                         cutDirs: 3,
1652                 },
1653         } {
1654                 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1655                 resp := httptest.NewRecorder()
1656                 u := mustParseURL("//" + trial.uri)
1657                 req := &http.Request{
1658                         Method:     "GET",
1659                         Host:       u.Host,
1660                         URL:        u,
1661                         RequestURI: u.RequestURI(),
1662                         Header:     copyHeader(trial.header),
1663                 }
1664                 s.handler.ServeHTTP(resp, req)
1665                 var cookies []*http.Cookie
1666                 for resp.Code == http.StatusSeeOther {
1667                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1668                         req = &http.Request{
1669                                 Method:     "GET",
1670                                 Host:       u.Host,
1671                                 URL:        u,
1672                                 RequestURI: u.RequestURI(),
1673                                 Header:     copyHeader(trial.header),
1674                         }
1675                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1676                         for _, c := range cookies {
1677                                 req.AddCookie(c)
1678                         }
1679                         resp = httptest.NewRecorder()
1680                         s.handler.ServeHTTP(resp, req)
1681                 }
1682                 if trial.redirect != "" {
1683                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1684                 }
1685                 if trial.expect == nil {
1686                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1687                 } else {
1688                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1689                         listingPageDoc, err := html.Parse(resp.Body)
1690                         c.Check(err, check.IsNil, comment) // valid HTML document
1691                         pathHrefMap := getPathHrefMap(listingPageDoc)
1692                         c.Assert(pathHrefMap, check.Not(check.HasLen), 0, comment)
1693                         for _, e := range trial.expect {
1694                                 href, hasE := pathHrefMap[e]
1695                                 c.Check(hasE, check.Equals, true, comment) // expected path is listed
1696                                 relUrl := mustParseURL(href)
1697                                 c.Check(relUrl.Path, check.Equals, "./"+e, comment) // href can be decoded back to path
1698                         }
1699                         wgetCommand := getWgetExamplePre(listingPageDoc)
1700                         wgetExpected := regexp.MustCompile(`^\$ wget .*--cut-dirs=(\d+) .*'(https?://[^']+)'$`)
1701                         wgetMatchGroups := wgetExpected.FindStringSubmatch(wgetCommand)
1702                         c.Assert(wgetMatchGroups, check.NotNil)                                     // wget command matches
1703                         c.Check(wgetMatchGroups[1], check.Equals, fmt.Sprintf("%d", trial.cutDirs)) // correct level of cut dirs in wget command
1704                         printedUrl := mustParseURL(wgetMatchGroups[2])
1705                         c.Check(printedUrl.Host, check.Equals, req.URL.Host)
1706                         c.Check(printedUrl.Path, check.Equals, req.URL.Path) // URL arg in wget command can be decoded to the right path
1707                 }
1708
1709                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1710                 req = &http.Request{
1711                         Method:     "OPTIONS",
1712                         Host:       u.Host,
1713                         URL:        u,
1714                         RequestURI: u.RequestURI(),
1715                         Header:     copyHeader(trial.header),
1716                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1717                 }
1718                 resp = httptest.NewRecorder()
1719                 s.handler.ServeHTTP(resp, req)
1720                 if trial.expect == nil {
1721                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1722                 } else {
1723                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1724                 }
1725
1726                 req = &http.Request{
1727                         Method:     "PROPFIND",
1728                         Host:       u.Host,
1729                         URL:        u,
1730                         RequestURI: u.RequestURI(),
1731                         Header:     copyHeader(trial.header),
1732                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1733                 }
1734                 resp = httptest.NewRecorder()
1735                 s.handler.ServeHTTP(resp, req)
1736                 // This check avoids logging a big XML document in the
1737                 // event webdav throws a 500 error after sending
1738                 // headers for a 207.
1739                 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1740                         continue
1741                 }
1742                 if trial.expect == nil {
1743                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1744                 } else {
1745                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1746                         for _, e := range trial.expect {
1747                                 if strings.HasSuffix(e, "/") {
1748                                         e = filepath.Join(u.Path, e) + "/"
1749                                 } else {
1750                                         e = filepath.Join(u.Path, e)
1751                                 }
1752                                 e = strings.Replace(e, " ", "%20", -1)
1753                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1754                         }
1755                 }
1756         }
1757 }
1758
1759 // Shallow-traverse the HTML document, gathering the nodes satisfying the
1760 // predicate function in the output slice. If a node matches the predicate,
1761 // none of its children will be visited.
1762 func getNodes(document *html.Node, predicate func(*html.Node) bool) []*html.Node {
1763         var acc []*html.Node
1764         var traverse func(*html.Node, []*html.Node) []*html.Node
1765         traverse = func(root *html.Node, sofar []*html.Node) []*html.Node {
1766                 if root == nil {
1767                         return sofar
1768                 }
1769                 if predicate(root) {
1770                         return append(sofar, root)
1771                 }
1772                 for cur := root.FirstChild; cur != nil; cur = cur.NextSibling {
1773                         sofar = traverse(cur, sofar)
1774                 }
1775                 return sofar
1776         }
1777         return traverse(document, acc)
1778 }
1779
1780 // Returns true if a node has the attribute targetAttr with the given value
1781 func matchesAttributeValue(node *html.Node, targetAttr string, value string) bool {
1782         for _, attr := range node.Attr {
1783                 if attr.Key == targetAttr && attr.Val == value {
1784                         return true
1785                 }
1786         }
1787         return false
1788 }
1789
1790 // Concatenate the content of text-node children of node; only direct
1791 // children are visited, and any non-text children are skipped.
1792 func getNodeText(node *html.Node) string {
1793         var recv strings.Builder
1794         for c := node.FirstChild; c != nil; c = c.NextSibling {
1795                 if c.Type == html.TextNode {
1796                         recv.WriteString(c.Data)
1797                 }
1798         }
1799         return recv.String()
1800 }
1801
1802 // Returns a map from the directory listing item string (a path) to the href
1803 // value of its <a> tag (an encoded relative URL)
1804 func getPathHrefMap(document *html.Node) map[string]string {
1805         isItemATag := func(node *html.Node) bool {
1806                 return node.Type == html.ElementNode && node.Data == "a" && matchesAttributeValue(node, "class", "item")
1807         }
1808         aTags := getNodes(document, isItemATag)
1809         output := make(map[string]string)
1810         for _, elem := range aTags {
1811                 textContent := getNodeText(elem)
1812                 for _, attr := range elem.Attr {
1813                         if attr.Key == "href" {
1814                                 output[textContent] = attr.Val
1815                                 break
1816                         }
1817                 }
1818         }
1819         return output
1820 }
1821
1822 func getWgetExamplePre(document *html.Node) string {
1823         isWgetPre := func(node *html.Node) bool {
1824                 return node.Type == html.ElementNode && matchesAttributeValue(node, "id", "wget-example")
1825         }
1826         elements := getNodes(document, isWgetPre)
1827         if len(elements) != 1 {
1828                 return ""
1829         }
1830         return getNodeText(elements[0])
1831 }
1832
1833 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1834         arv := arvados.NewClientFromEnv()
1835         var newCollection arvados.Collection
1836         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1837                 "collection": map[string]string{
1838                         "owner_uuid":    arvadostest.ActiveUserUUID,
1839                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1840                         "name":          "keep-web test collection",
1841                 },
1842                 "ensure_unique_name": true,
1843         })
1844         c.Assert(err, check.IsNil)
1845         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1846
1847         var updated arvados.Collection
1848         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1849                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1850                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1851                 req := &http.Request{
1852                         Method:     "DELETE",
1853                         Host:       u.Host,
1854                         URL:        u,
1855                         RequestURI: u.RequestURI(),
1856                         Header: http.Header{
1857                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1858                         },
1859                 }
1860                 resp := httptest.NewRecorder()
1861                 s.handler.ServeHTTP(resp, req)
1862                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1863
1864                 updated = arvados.Collection{}
1865                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1866                 c.Check(err, check.IsNil)
1867                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1868                 c.Logf("updated manifest_text %q", updated.ManifestText)
1869         }
1870         c.Check(updated.ManifestText, check.Equals, "")
1871 }
1872
1873 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1874         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1875
1876         client := arvados.NewClientFromEnv()
1877         client.AuthToken = arvadostest.ActiveToken
1878         arv, err := arvadosclient.New(client)
1879         c.Assert(err, check.Equals, nil)
1880         kc, err := keepclient.MakeKeepClient(arv)
1881         c.Assert(err, check.Equals, nil)
1882
1883         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1884         c.Assert(err, check.IsNil)
1885
1886         trials := []struct {
1887                 filename    string
1888                 content     string
1889                 contentType string
1890         }{
1891                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1892                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1893                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1894                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1895                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1896         }
1897         for _, trial := range trials {
1898                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1899                 c.Assert(err, check.IsNil)
1900                 _, err = f.Write([]byte(trial.content))
1901                 c.Assert(err, check.IsNil)
1902                 c.Assert(f.Close(), check.IsNil)
1903         }
1904         mtxt, err := fs.MarshalManifest(".")
1905         c.Assert(err, check.IsNil)
1906         var coll arvados.Collection
1907         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1908                 "collection": map[string]string{
1909                         "manifest_text": mtxt,
1910                 },
1911         })
1912         c.Assert(err, check.IsNil)
1913
1914         for _, trial := range trials {
1915                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1916                 req := &http.Request{
1917                         Method:     "GET",
1918                         Host:       u.Host,
1919                         URL:        u,
1920                         RequestURI: u.RequestURI(),
1921                         Header: http.Header{
1922                                 "Authorization": {"Bearer " + client.AuthToken},
1923                         },
1924                 }
1925                 resp := httptest.NewRecorder()
1926                 s.handler.ServeHTTP(resp, req)
1927                 c.Check(resp.Code, check.Equals, http.StatusOK)
1928                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1929                 c.Check(resp.Body.String(), check.Equals, trial.content)
1930         }
1931 }
1932
1933 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1934         req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1935         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1936         c.Assert(err, check.IsNil)
1937         resp := httptest.NewRecorder()
1938         s.handler.ServeHTTP(resp, req)
1939         c.Assert(resp.Code, check.Equals, http.StatusOK)
1940         c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1941 }
1942
1943 // Writing to a collection shouldn't affect its entry in the
1944 // PDH-to-manifest cache.
1945 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1946         arv, err := arvadosclient.MakeArvadosClient()
1947         c.Assert(err, check.Equals, nil)
1948         arv.ApiToken = arvadostest.ActiveToken
1949
1950         u := mustParseURL("http://x.example/testfile")
1951         req := &http.Request{
1952                 Method:     "GET",
1953                 Host:       u.Host,
1954                 URL:        u,
1955                 RequestURI: u.RequestURI(),
1956                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1957         }
1958
1959         checkWithID := func(id string, status int) {
1960                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1961                 req.Host = req.URL.Host
1962                 resp := httptest.NewRecorder()
1963                 s.handler.ServeHTTP(resp, req)
1964                 c.Check(resp.Code, check.Equals, status)
1965         }
1966
1967         var colls [2]arvados.Collection
1968         for i := range colls {
1969                 err := arv.Create("collections",
1970                         map[string]interface{}{
1971                                 "ensure_unique_name": true,
1972                                 "collection": map[string]interface{}{
1973                                         "name": "test collection",
1974                                 },
1975                         }, &colls[i])
1976                 c.Assert(err, check.Equals, nil)
1977         }
1978
1979         // Populate cache with empty collection
1980         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1981
1982         // write a file to colls[0]
1983         reqPut := *req
1984         reqPut.Method = "PUT"
1985         reqPut.URL.Host = colls[0].UUID + ".example"
1986         reqPut.Host = req.URL.Host
1987         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1988         resp := httptest.NewRecorder()
1989         s.handler.ServeHTTP(resp, &reqPut)
1990         c.Check(resp.Code, check.Equals, http.StatusCreated)
1991
1992         // new file should not appear in colls[1]
1993         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1994         checkWithID(colls[1].UUID, http.StatusNotFound)
1995
1996         checkWithID(colls[0].UUID, http.StatusOK)
1997 }
1998
1999 func copyHeader(h http.Header) http.Header {
2000         hc := http.Header{}
2001         for k, v := range h {
2002                 hc[k] = append([]string(nil), v...)
2003         }
2004         return hc
2005 }
2006
2007 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
2008         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
2009
2010         client := arvados.NewClientFromEnv()
2011         client.AuthToken = arvadostest.AdminToken
2012         var logentries arvados.LogList
2013         limit1 := 1
2014         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
2015                 arvados.ResourceListParams{
2016                         Limit: &limit1,
2017                         Order: "created_at desc"})
2018         c.Check(err, check.IsNil)
2019         c.Check(logentries.Items, check.HasLen, 1)
2020         lastLogId := logentries.Items[0].ID
2021         c.Logf("lastLogId: %d", lastLogId)
2022
2023         var logbuf bytes.Buffer
2024         logger := logrus.New()
2025         logger.Out = &logbuf
2026         resp := httptest.NewRecorder()
2027         req = req.WithContext(ctxlog.Context(context.Background(), logger))
2028         s.handler.ServeHTTP(resp, req)
2029
2030         if perm {
2031                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
2032                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
2033                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
2034
2035                 deadline := time.Now().Add(time.Second)
2036                 for {
2037                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
2038                         logentries = arvados.LogList{}
2039                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
2040                                 arvados.ResourceListParams{
2041                                         Filters: []arvados.Filter{
2042                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
2043                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
2044                                         },
2045                                         Limit: &limit1,
2046                                         Order: "created_at desc",
2047                                 })
2048                         c.Assert(err, check.IsNil)
2049                         if len(logentries.Items) > 0 &&
2050                                 logentries.Items[0].ID > lastLogId &&
2051                                 logentries.Items[0].ObjectUUID == userUuid &&
2052                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
2053                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
2054                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
2055                                 break
2056                         }
2057                         c.Logf("logentries.Items: %+v", logentries.Items)
2058                         time.Sleep(50 * time.Millisecond)
2059                 }
2060         } else {
2061                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
2062                 c.Check(logbuf.String(), check.Equals, "")
2063         }
2064 }
2065
2066 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
2067         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
2068
2069         s.handler.Cluster.Collections.TrustAllContent = true
2070         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(0)
2071
2072         for _, adminperm := range []bool{true, false} {
2073                 for _, userperm := range []bool{true, false} {
2074                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
2075                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
2076
2077                         // Test admin permission
2078                         req := &http.Request{
2079                                 Method:     "GET",
2080                                 Host:       u.Host,
2081                                 URL:        u,
2082                                 RequestURI: u.RequestURI(),
2083                                 Header: http.Header{
2084                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
2085                                 },
2086                         }
2087                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
2088                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
2089
2090                         // Test user permission
2091                         req = &http.Request{
2092                                 Method:     "GET",
2093                                 Host:       u.Host,
2094                                 URL:        u,
2095                                 RequestURI: u.RequestURI(),
2096                                 Header: http.Header{
2097                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2098                                 },
2099                         }
2100                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
2101                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
2102                 }
2103         }
2104
2105         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
2106
2107         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
2108                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
2109
2110                 u = mustParseURL(tryurl)
2111                 req := &http.Request{
2112                         Method:     "GET",
2113                         Host:       u.Host,
2114                         URL:        u,
2115                         RequestURI: u.RequestURI(),
2116                         Header: http.Header{
2117                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
2118                         },
2119                 }
2120                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
2121                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
2122         }
2123
2124         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
2125         req := &http.Request{
2126                 Method:     "GET",
2127                 Host:       u.Host,
2128                 URL:        u,
2129                 RequestURI: u.RequestURI(),
2130                 Header: http.Header{
2131                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2132                 },
2133         }
2134         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
2135                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
2136 }
2137
2138 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
2139         for _, adminperm := range []bool{true, false} {
2140                 for _, userperm := range []bool{true, false} {
2141
2142                         arv := arvados.NewClientFromEnv()
2143                         arv.AuthToken = arvadostest.ActiveToken
2144
2145                         var coll arvados.Collection
2146                         err := arv.RequestAndDecode(&coll,
2147                                 "POST",
2148                                 "/arvados/v1/collections",
2149                                 nil,
2150                                 map[string]interface{}{
2151                                         "ensure_unique_name": true,
2152                                         "collection": map[string]interface{}{
2153                                                 "name": "test collection",
2154                                         },
2155                                 })
2156                         c.Assert(err, check.Equals, nil)
2157
2158                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
2159
2160                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
2161                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
2162
2163                         // Test admin permission
2164                         req := &http.Request{
2165                                 Method:     "PUT",
2166                                 Host:       u.Host,
2167                                 URL:        u,
2168                                 RequestURI: u.RequestURI(),
2169                                 Header: http.Header{
2170                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
2171                                 },
2172                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
2173                         }
2174                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
2175                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
2176
2177                         // Test user permission
2178                         req = &http.Request{
2179                                 Method:     "PUT",
2180                                 Host:       u.Host,
2181                                 URL:        u,
2182                                 RequestURI: u.RequestURI(),
2183                                 Header: http.Header{
2184                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2185                                 },
2186                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
2187                         }
2188                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
2189                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
2190                 }
2191         }
2192 }
2193
2194 func (s *IntegrationSuite) TestLogThrottling(c *check.C) {
2195         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Hour)
2196         logbuf, ctx := newLoggerAndContext()
2197         fooURL := "http://" + arvadostest.FooCollection + ".keep-web.example/foo"
2198
2199         var wg sync.WaitGroup
2200         for _, byterange := range []string{"0-2", "0-1", "1-2"} {
2201                 byterange := byterange
2202                 wg.Add(1)
2203                 go func() {
2204                         defer wg.Done()
2205                         req := newRequest("GET", fooURL)
2206                         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
2207                         req.Header.Set("Range", "bytes="+byterange)
2208                         req = req.WithContext(ctx)
2209                         resp := httptest.NewRecorder()
2210                         s.handler.ServeHTTP(resp, req)
2211                         c.Check(resp.Result().StatusCode, check.Equals, http.StatusPartialContent)
2212                 }()
2213         }
2214         wg.Wait()
2215
2216         re := regexp.MustCompile(`\bmsg="File download".* collection_file_path=foo\b`)
2217         matches := re.FindAll(logbuf.Bytes(), -1)
2218         c.Check(matches, check.HasLen, 1,
2219                 check.Commentf("%d matching log messages: %+v", len(matches), matches))
2220 }
2221
2222 func (s *IntegrationSuite) TestLogThrottleInterval(c *check.C) {
2223         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Nanosecond)
2224         logbuf, ctx := newLoggerAndContext()
2225         req := newRequest("GET", "http://"+arvadostest.FooCollection+".keep-web.example/foo")
2226         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
2227         req = req.WithContext(ctx)
2228
2229         re := regexp.MustCompile(`\bmsg="File download".* collection_file_path=foo\b`)
2230         for expected := 1; expected < 4; expected++ {
2231                 time.Sleep(2 * time.Nanosecond)
2232                 resp := httptest.NewRecorder()
2233                 s.handler.ServeHTTP(resp, req)
2234                 c.Assert(resp.Result().StatusCode, check.Equals, http.StatusOK)
2235                 matches := re.FindAll(logbuf.Bytes(), -1)
2236                 c.Assert(matches, check.HasLen, expected,
2237                         check.Commentf("%d matching log messages: %+v", len(matches), matches))
2238         }
2239 }
2240
2241 func (s *IntegrationSuite) TestLogThrottleDifferentTokens(c *check.C) {
2242         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Hour)
2243         logbuf, ctx := newLoggerAndContext()
2244         req := newRequest("GET", "http://"+arvadostest.FooCollection+".keep-web.example/foo")
2245         req = req.WithContext(ctx)
2246
2247         trials := []string{arvadostest.ActiveToken, arvadostest.AdminToken}
2248         for _, token := range trials {
2249                 req.Header.Set("Authorization", "Bearer "+token)
2250                 resp := httptest.NewRecorder()
2251                 s.handler.ServeHTTP(resp, req)
2252                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusOK)
2253         }
2254
2255         re := regexp.MustCompile(`\bmsg="File download".* collection_file_path=foo\b`)
2256         matches := re.FindAll(logbuf.Bytes(), -1)
2257         c.Check(matches, check.HasLen, len(trials),
2258                 check.Commentf("%d matching log messages: %+v", len(matches), matches))
2259 }
2260
2261 func (s *IntegrationSuite) TestLogThrottleDifferentFiles(c *check.C) {
2262         s.handler.Cluster.Collections.WebDAVLogDownloadInterval = arvados.Duration(time.Hour)
2263         logbuf, ctx := newLoggerAndContext()
2264         baseURL := "http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/"
2265
2266         trials := []string{"file1", "file2", "file3"}
2267         for _, filename := range trials {
2268                 req := newRequest("GET", baseURL+filename)
2269                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
2270                 req = req.WithContext(ctx)
2271                 resp := httptest.NewRecorder()
2272                 s.handler.ServeHTTP(resp, req)
2273                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusOK)
2274         }
2275
2276         re := regexp.MustCompile(`\bmsg="File download".* collection_uuid=` + arvadostest.MultilevelCollection1 + `\b`)
2277         matches := re.FindAll(logbuf.Bytes(), -1)
2278         c.Check(matches, check.HasLen, len(trials),
2279                 check.Commentf("%d matching log messages: %+v", len(matches), matches))
2280 }
2281
2282 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
2283         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
2284         lockTidyInterval = time.Second
2285         client := arvados.NewClientFromEnv()
2286         client.AuthToken = arvadostest.ActiveTokenV2
2287         // Start small, and increase concurrency (2^2, 4^2, ...)
2288         // only until hitting failure. Avoids unnecessarily long
2289         // failure reports.
2290         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
2291                 c.Logf("%s: n=%d", c.TestName(), n)
2292
2293                 var coll arvados.Collection
2294                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
2295                 c.Assert(err, check.IsNil)
2296                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
2297
2298                 var wg sync.WaitGroup
2299                 for i := 0; i < n && !c.Failed(); i++ {
2300                         i := i
2301                         wg.Add(1)
2302                         go func() {
2303                                 defer wg.Done()
2304                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
2305                                 resp := httptest.NewRecorder()
2306                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
2307                                 c.Assert(err, check.IsNil)
2308                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2309                                 s.handler.ServeHTTP(resp, req)
2310                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
2311                                 for j := 0; j < n && !c.Failed(); j++ {
2312                                         j := j
2313                                         wg.Add(1)
2314                                         go func() {
2315                                                 defer wg.Done()
2316                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
2317                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
2318
2319                                                 resp := httptest.NewRecorder()
2320                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
2321                                                 c.Assert(err, check.IsNil)
2322                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2323                                                 s.handler.ServeHTTP(resp, req)
2324                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
2325
2326                                                 time.Sleep(time.Second)
2327                                                 resp = httptest.NewRecorder()
2328                                                 req, err = http.NewRequest("GET", u.String(), nil)
2329                                                 c.Assert(err, check.IsNil)
2330                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2331                                                 s.handler.ServeHTTP(resp, req)
2332                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
2333                                                 c.Check(resp.Body.String(), check.Equals, content)
2334                                         }()
2335                                 }
2336                         }()
2337                 }
2338                 wg.Wait()
2339                 for i := 0; i < n; i++ {
2340                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
2341                         resp := httptest.NewRecorder()
2342                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
2343                         c.Assert(err, check.IsNil)
2344                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2345                         s.handler.ServeHTTP(resp, req)
2346                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
2347                 }
2348         }
2349 }