services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "sync"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         "golang.org/x/net/html"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63                 metrics: newMetrics(prometheus.NewRegistry()),
  64         }
  65 }
  66
  67 func newCollection(collID string) *arvados.Collection {
  68         coll := &arvados.Collection{UUID: collID}
  69         if pdh, ok := arvadostest.TestCollectionUUIDToPDH[collID]; ok {
  70                 coll.PortableDataHash = pdh
  71         }
  72         return coll
  73 }
  74
  75 func newRequest(method, urlStr string) *http.Request {
  76         u := mustParseURL(urlStr)
  77         return &http.Request{
  78                 Method:     method,
  79                 Host:       u.Host,
  80                 URL:        u,
  81                 RequestURI: u.RequestURI(),
  82                 RemoteAddr: "10.20.30.40:56789",
  83                 Header:     http.Header{},
  84         }
  85 }
  86
  87 func (s *UnitSuite) TestLogEventTypes(c *check.C) {
  88         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
  89         for method, expected := range map[string]string{
  90                 "GET":  "file_download",
  91                 "POST": "file_upload",
  92                 "PUT":  "file_upload",
  93         } {
  94                 filePath := "/" + method
  95                 req := newRequest(method, collURL+filePath)
  96                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
  97                 if !c.Check(actual, check.NotNil) {
  98                         continue
  99                 }
 100                 c.Check(actual.eventType, check.Equals, expected)
 101         }
 102 }
 103
 104 func (s *UnitSuite) TestUnloggedEventTypes(c *check.C) {
 105         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 106         for _, method := range []string{"DELETE", "HEAD", "OPTIONS", "PATCH"} {
 107                 filePath := "/" + method
 108                 req := newRequest(method, collURL+filePath)
 109                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 110                 c.Check(actual, check.IsNil,
 111                         check.Commentf("%s request made a log event", method))
 112         }
 113 }
 114
 115 func (s *UnitSuite) TestLogFilePath(c *check.C) {
 116         coll := newCollection(arvadostest.FooCollection)
 117         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 118         for _, filePath := range []string{"/foo", "/Foo", "/foo/bar"} {
 119                 req := newRequest("GET", collURL+filePath)
 120                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 121                 if !c.Check(actual, check.NotNil) {
 122                         continue
 123                 }
 124                 c.Check(actual.collFilePath, check.Equals, filePath)
 125         }
 126 }
 127
 128 func (s *UnitSuite) TestLogRemoteAddr(c *check.C) {
 129         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 130         filePath := "/foo"
 131         req := newRequest("GET", collURL+filePath)
 132
 133         for _, addr := range []string{"10.20.30.55", "192.168.144.120", "192.0.2.4"} {
 134                 req.RemoteAddr = addr + ":57914"
 135                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 136                 if !c.Check(actual, check.NotNil) {
 137                         continue
 138                 }
 139                 c.Check(actual.clientAddr, check.Equals, addr)
 140         }
 141
 142         for _, addr := range []string{"100::20:30:40", "2001:db8::90:100", "3fff::30"} {
 143                 req.RemoteAddr = fmt.Sprintf("[%s]:57916", addr)
 144                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 145                 if !c.Check(actual, check.NotNil) {
 146                         continue
 147                 }
 148                 c.Check(actual.clientAddr, check.Equals, addr)
 149         }
 150 }
 151
 152 func (s *UnitSuite) TestLogXForwardedFor(c *check.C) {
 153         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 154         filePath := "/foo"
 155         req := newRequest("GET", collURL+filePath)
 156         for xff, expected := range map[string]string{
 157                 "10.20.30.55":                          "10.20.30.55",
 158                 "192.168.144.120, 10.20.30.120":        "192.168.144.120",
 159                 "192.0.2.4, 192.0.2.6, 192.0.2.8":      "192.0.2.4",
 160                 "192.0.2.4,192.168.2.4":                "192.0.2.4",
 161                 "10.20.30.60,192.168.144.40,192.0.2.4": "10.20.30.60",
 162                 "100::20:30:50":                        "100::20:30:50",
 163                 "2001:db8::80:90, 100::100":            "2001:db8::80:90",
 164                 "3fff::ff, 3fff::ee, 3fff::fe":         "3fff::ff",
 165                 "3fff::3f,100::1000":                   "3fff::3f",
 166                 "2001:db8::88,100::88,3fff::88":        "2001:db8::88",
 167                 "10.20.30.60, 2001:db8::60":            "10.20.30.60",
 168                 "2001:db8::20,10.20.30.20":             "2001:db8::20",
 169                 ", 10.20.30.123, 100::123":             "10.20.30.123",
 170                 ",100::321,10.30.20.10":                "100::321",
 171         } {
 172                 req.Header.Set("X-Forwarded-For", xff)
 173                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 174                 if !c.Check(actual, check.NotNil) {
 175                         continue
 176                 }
 177                 c.Check(actual.clientAddr, check.Equals, expected)
 178         }
 179 }
 180
 181 func (s *UnitSuite) TestLogXForwardedForMalformed(c *check.C) {
 182         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 183         filePath := "/foo"
 184         req := newRequest("GET", collURL+filePath)
 185         for _, xff := range []string{"", ",", "10.20,30.40", "foo, bar"} {
 186                 req.Header.Set("X-Forwarded-For", xff)
 187                 actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 188                 if !c.Check(actual, check.NotNil) {
 189                         continue
 190                 }
 191                 c.Check(actual.clientAddr, check.Equals, "10.20.30.40")
 192         }
 193 }
 194
 195 func (s *UnitSuite) TestLogXForwardedForMultivalue(c *check.C) {
 196         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 197         filePath := "/foo"
 198         req := newRequest("GET", collURL+filePath)
 199         req.Header.Set("X-Forwarded-For", ", ")
 200         req.Header.Add("X-Forwarded-For", "2001:db8::db9:dbd")
 201         req.Header.Add("X-Forwarded-For", "10.20.30.90")
 202         actual := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 203         c.Assert(actual, check.NotNil)
 204         c.Check(actual.clientAddr, check.Equals, "2001:db8::db9:dbd")
 205 }
 206
 207 func (s *UnitSuite) TestLogClientAddressCanonicalization(c *check.C) {
 208         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 209         filePath := "/foo"
 210         req := newRequest("GET", collURL+filePath)
 211         expected := "2001:db8::12:0"
 212
 213         req.RemoteAddr = "[2001:db8::012:0000]:57918"
 214         a := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 215         c.Assert(a, check.NotNil)
 216         c.Check(a.clientAddr, check.Equals, expected)
 217
 218         req.RemoteAddr = "10.20.30.40:57919"
 219         req.Header.Set("X-Forwarded-For", "2001:db8:0::0:12:00")
 220         b := newFileEventLog(s.handler, req, filePath, nil, nil, "")
 221         c.Assert(b, check.NotNil)
 222         c.Check(b.clientAddr, check.Equals, expected)
 223 }
 224
 225 func (s *UnitSuite) TestLogAnonymousUser(c *check.C) {
 226         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 227         filePath := "/foo"
 228         req := newRequest("GET", collURL+filePath)
 229         actual := newFileEventLog(s.handler, req, filePath, nil, nil, arvadostest.AnonymousToken)
 230         c.Assert(actual, check.NotNil)
 231         c.Check(actual.userUUID, check.Equals, s.handler.Cluster.ClusterID+"-tpzed-anonymouspublic")
 232         c.Check(actual.userFullName, check.Equals, "")
 233         c.Check(actual.clientToken, check.Equals, arvadostest.AnonymousToken)
 234 }
 235
 236 func (s *UnitSuite) TestLogUser(c *check.C) {
 237         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 238         for _, trial := range []struct{ uuid, fullName, token string }{
 239                 {arvadostest.ActiveUserUUID, "Active User", arvadostest.ActiveToken},
 240                 {arvadostest.SpectatorUserUUID, "Spectator User", arvadostest.SpectatorToken},
 241         } {
 242                 filePath := "/" + trial.uuid
 243                 req := newRequest("GET", collURL+filePath)
 244                 user := &arvados.User{
 245                         UUID:     trial.uuid,
 246                         FullName: trial.fullName,
 247                 }
 248                 actual := newFileEventLog(s.handler, req, filePath, nil, user, trial.token)
 249                 if !c.Check(actual, check.NotNil) {
 250                         continue
 251                 }
 252                 c.Check(actual.userUUID, check.Equals, trial.uuid)
 253                 c.Check(actual.userFullName, check.Equals, trial.fullName)
 254                 c.Check(actual.clientToken, check.Equals, trial.token)
 255         }
 256 }
 257
 258 func (s *UnitSuite) TestLogCollectionByUUID(c *check.C) {
 259         for collUUID, collPDH := range arvadostest.TestCollectionUUIDToPDH {
 260                 collURL := "http://keep-web.example/c=" + collUUID
 261                 filePath := "/" + collUUID
 262                 req := newRequest("GET", collURL+filePath)
 263                 coll := newCollection(collUUID)
 264                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 265                 if !c.Check(actual, check.NotNil) {
 266                         continue
 267                 }
 268                 c.Check(actual.collUUID, check.Equals, collUUID)
 269                 c.Check(actual.collPDH, check.Equals, collPDH)
 270         }
 271 }
 272
 273 func (s *UnitSuite) TestLogCollectionByPDH(c *check.C) {
 274         for _, collPDH := range arvadostest.TestCollectionUUIDToPDH {
 275                 collURL := "http://keep-web.example/c=" + collPDH
 276                 filePath := "/PDHFile"
 277                 req := newRequest("GET", collURL+filePath)
 278                 coll := newCollection(collPDH)
 279                 actual := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 280                 if !c.Check(actual, check.NotNil) {
 281                         continue
 282                 }
 283                 c.Check(actual.collPDH, check.Equals, collPDH)
 284                 c.Check(actual.collUUID, check.Equals, "")
 285         }
 286 }
 287
 288 func (s *UnitSuite) TestLogGETUUIDAsDict(c *check.C) {
 289         filePath := "/foo"
 290         reqPath := "/c=" + arvadostest.FooCollection + filePath
 291         req := newRequest("GET", "http://keep-web.example"+reqPath)
 292         coll := newCollection(arvadostest.FooCollection)
 293         logEvent := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 294         c.Assert(logEvent, check.NotNil)
 295         c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 296                 "event_type":  "file_download",
 297                 "object_uuid": s.handler.Cluster.ClusterID + "-tpzed-anonymouspublic",
 298                 "properties": arvadosclient.Dict{
 299                         "reqPath":              reqPath,
 300                         "collection_uuid":      arvadostest.FooCollection,
 301                         "collection_file_path": filePath,
 302                         "portable_data_hash":   arvadostest.FooCollectionPDH,
 303                 },
 304         })
 305 }
 306
 307 func (s *UnitSuite) TestLogGETPDHAsDict(c *check.C) {
 308         filePath := "/Foo"
 309         reqPath := "/c=" + arvadostest.FooCollectionPDH + filePath
 310         req := newRequest("GET", "http://keep-web.example"+reqPath)
 311         coll := newCollection(arvadostest.FooCollectionPDH)
 312         user := &arvados.User{
 313                 UUID:     arvadostest.ActiveUserUUID,
 314                 FullName: "Active User",
 315         }
 316         logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 317         c.Assert(logEvent, check.NotNil)
 318         c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 319                 "event_type":  "file_download",
 320                 "object_uuid": arvadostest.ActiveUserUUID,
 321                 "properties": arvadosclient.Dict{
 322                         "reqPath":              reqPath,
 323                         "portable_data_hash":   arvadostest.FooCollectionPDH,
 324                         "collection_uuid":      "",
 325                         "collection_file_path": filePath,
 326                 },
 327         })
 328 }
 329
 330 func (s *UnitSuite) TestLogUploadAsDict(c *check.C) {
 331         coll := newCollection(arvadostest.FooCollection)
 332         user := &arvados.User{
 333                 UUID:     arvadostest.ActiveUserUUID,
 334                 FullName: "Active User",
 335         }
 336         for _, method := range []string{"POST", "PUT"} {
 337                 filePath := "/" + method + "File"
 338                 reqPath := "/c=" + arvadostest.FooCollection + filePath
 339                 req := newRequest(method, "http://keep-web.example"+reqPath)
 340                 logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 341                 if !c.Check(logEvent, check.NotNil) {
 342                         continue
 343                 }
 344                 c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 345                         "event_type":  "file_upload",
 346                         "object_uuid": arvadostest.ActiveUserUUID,
 347                         "properties": arvadosclient.Dict{
 348                                 "reqPath":              reqPath,
 349                                 "collection_uuid":      arvadostest.FooCollection,
 350                                 "collection_file_path": filePath,
 351                         },
 352                 })
 353         }
 354 }
 355
 356 func (s *UnitSuite) TestLogGETUUIDAsFields(c *check.C) {
 357         filePath := "/foo"
 358         reqPath := "/c=" + arvadostest.FooCollection + filePath
 359         req := newRequest("GET", "http://keep-web.example"+reqPath)
 360         coll := newCollection(arvadostest.FooCollection)
 361         logEvent := newFileEventLog(s.handler, req, filePath, coll, nil, "")
 362         c.Assert(logEvent, check.NotNil)
 363         c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 364                 "user_uuid":            s.handler.Cluster.ClusterID + "-tpzed-anonymouspublic",
 365                 "collection_uuid":      arvadostest.FooCollection,
 366                 "collection_file_path": filePath,
 367                 "portable_data_hash":   arvadostest.FooCollectionPDH,
 368         })
 369 }
 370
 371 func (s *UnitSuite) TestLogGETPDHAsFields(c *check.C) {
 372         filePath := "/Foo"
 373         reqPath := "/c=" + arvadostest.FooCollectionPDH + filePath
 374         req := newRequest("GET", "http://keep-web.example"+reqPath)
 375         coll := newCollection(arvadostest.FooCollectionPDH)
 376         user := &arvados.User{
 377                 UUID:     arvadostest.ActiveUserUUID,
 378                 FullName: "Active User",
 379         }
 380         logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 381         c.Assert(logEvent, check.NotNil)
 382         c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 383                 "user_uuid":            arvadostest.ActiveUserUUID,
 384                 "user_full_name":       "Active User",
 385                 "collection_uuid":      "",
 386                 "collection_file_path": filePath,
 387                 "portable_data_hash":   arvadostest.FooCollectionPDH,
 388         })
 389 }
 390
 391 func (s *UnitSuite) TestLogUploadAsFields(c *check.C) {
 392         coll := newCollection(arvadostest.FooCollection)
 393         user := &arvados.User{
 394                 UUID:     arvadostest.ActiveUserUUID,
 395                 FullName: "Active User",
 396         }
 397         for _, method := range []string{"POST", "PUT"} {
 398                 filePath := "/" + method + "File"
 399                 reqPath := "/c=" + arvadostest.FooCollection + filePath
 400                 req := newRequest(method, "http://keep-web.example"+reqPath)
 401                 logEvent := newFileEventLog(s.handler, req, filePath, coll, user, "")
 402                 if !c.Check(logEvent, check.NotNil) {
 403                         continue
 404                 }
 405                 c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 406                         "user_uuid":            arvadostest.ActiveUserUUID,
 407                         "user_full_name":       "Active User",
 408                         "collection_uuid":      arvadostest.FooCollection,
 409                         "collection_file_path": filePath,
 410                 })
 411         }
 412 }
 413
 414 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
 415         h := s.handler
 416         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
 417         req := &http.Request{
 418                 Method:     "OPTIONS",
 419                 Host:       u.Host,
 420                 URL:        u,
 421                 RequestURI: u.RequestURI(),
 422                 Header: http.Header{
 423                         "Origin":                        {"https://workbench.example"},
 424                         "Access-Control-Request-Method": {"POST"},
 425                 },
 426         }
 427
 428         // Check preflight for an allowed request
 429         resp := httptest.NewRecorder()
 430         h.ServeHTTP(resp, req)
 431         c.Check(resp.Code, check.Equals, http.StatusOK)
 432         c.Check(resp.Body.String(), check.Equals, "")
 433         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 434         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
 435         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
 436
 437         // Check preflight for a disallowed request
 438         resp = httptest.NewRecorder()
 439         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
 440         h.ServeHTTP(resp, req)
 441         c.Check(resp.Body.String(), check.Equals, "")
 442         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
 443 }
 444
 445 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
 446         for _, trial := range []struct {
 447                 method   string
 448                 path     string
 449                 prefix   string
 450                 source   string
 451                 notFound bool
 452                 seeOther bool
 453         }{
 454                 {
 455                         method: "PROPFIND",
 456                         path:   "/",
 457                 },
 458                 {
 459                         method: "PROPFIND",
 460                         path:   "/dir1",
 461                 },
 462                 {
 463                         method: "PROPFIND",
 464                         path:   "/dir1/",
 465                 },
 466                 {
 467                         method: "PROPFIND",
 468                         path:   "/dir1/foo",
 469                         prefix: "/dir1",
 470                         source: "/dir1",
 471                 },
 472                 {
 473                         method: "PROPFIND",
 474                         path:   "/prefix/dir1/foo",
 475                         prefix: "/prefix/",
 476                         source: "",
 477                 },
 478                 {
 479                         method: "PROPFIND",
 480                         path:   "/prefix/dir1/foo",
 481                         prefix: "/prefix",
 482                         source: "",
 483                 },
 484                 {
 485                         method: "PROPFIND",
 486                         path:   "/prefix/dir1/foo",
 487                         prefix: "/prefix/",
 488                         source: "/",
 489                 },
 490                 {
 491                         method: "PROPFIND",
 492                         path:   "/prefix/foo",
 493                         prefix: "/prefix/",
 494                         source: "/dir1/",
 495                 },
 496                 {
 497                         method: "GET",
 498                         path:   "/prefix/foo",
 499                         prefix: "/prefix/",
 500                         source: "/dir1/",
 501                 },
 502                 {
 503                         method: "PROPFIND",
 504                         path:   "/prefix/",
 505                         prefix: "/prefix",
 506                         source: "/dir1",
 507                 },
 508                 {
 509                         method: "PROPFIND",
 510                         path:   "/prefix",
 511                         prefix: "/prefix",
 512                         source: "/dir1/",
 513                 },
 514                 {
 515                         method:   "GET",
 516                         path:     "/prefix",
 517                         prefix:   "/prefix",
 518                         source:   "/dir1",
 519                         seeOther: true,
 520                 },
 521                 {
 522                         method:   "PROPFIND",
 523                         path:     "/dir1/foo",
 524                         prefix:   "",
 525                         source:   "/dir1",
 526                         notFound: true,
 527                 },
 528         } {
 529                 c.Logf("trial %+v", trial)
 530                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 531                 req := &http.Request{
 532                         Method:     trial.method,
 533                         Host:       u.Host,
 534                         URL:        u,
 535                         RequestURI: u.RequestURI(),
 536                         Header: http.Header{
 537                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 538                                 "X-Webdav-Prefix": {trial.prefix},
 539                                 "X-Webdav-Source": {trial.source},
 540                         },
 541                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 542                 }
 543
 544                 resp := httptest.NewRecorder()
 545                 s.handler.ServeHTTP(resp, req)
 546                 if trial.notFound {
 547                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 548                 } else if trial.method == "PROPFIND" {
 549                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 550                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 551                 } else if trial.seeOther {
 552                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 553                 } else {
 554                         c.Check(resp.Code, check.Equals, http.StatusOK)
 555                 }
 556         }
 557 }
 558
 559 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 560         // Ensure we start with an empty cache
 561         defer os.Setenv("HOME", os.Getenv("HOME"))
 562         os.Setenv("HOME", c.MkDir())
 563
 564         for _, trial := range []struct {
 565                 dataExists    bool
 566                 sendIMSHeader bool
 567                 expectStatus  int
 568                 logRegexp     string
 569         }{
 570                 // If we return no content due to a Keep read error,
 571                 // we should emit a log message.
 572                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 573
 574                 // If we return no content because the client sent an
 575                 // If-Modified-Since header, our response should be
 576                 // 304.  We still expect a "File download" log since it
 577                 // counts as a file access for auditing.
 578                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 579         } {
 580                 c.Logf("trial: %+v", trial)
 581                 arvadostest.StartKeep(2, true)
 582                 if trial.dataExists {
 583                         arv, err := arvadosclient.MakeArvadosClient()
 584                         c.Assert(err, check.IsNil)
 585                         arv.ApiToken = arvadostest.ActiveToken
 586                         kc, err := keepclient.MakeKeepClient(arv)
 587                         c.Assert(err, check.IsNil)
 588                         _, _, err = kc.PutB([]byte("foo"))
 589                         c.Assert(err, check.IsNil)
 590                 }
 591
 592                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 593                 req := &http.Request{
 594                         Method:     "GET",
 595                         Host:       u.Host,
 596                         URL:        u,
 597                         RequestURI: u.RequestURI(),
 598                         Header: http.Header{
 599                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 600                         },
 601                 }
 602                 if trial.sendIMSHeader {
 603                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 604                 }
 605
 606                 var logbuf bytes.Buffer
 607                 logger := logrus.New()
 608                 logger.Out = &logbuf
 609                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 610
 611                 resp := httptest.NewRecorder()
 612                 s.handler.ServeHTTP(resp, req)
 613                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 614                 c.Check(resp.Body.String(), check.Equals, "")
 615
 616                 c.Log(logbuf.String())
 617                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 618         }
 619 }
 620
 621 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 622         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 623         token := arvadostest.ActiveToken
 624         for _, trial := range []string{
 625                 "http://keep-web/c=" + bogusID + "/foo",
 626                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 627                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 628                 "http://keep-web/collections/" + bogusID + "/foo",
 629                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 630                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 631         } {
 632                 c.Log(trial)
 633                 u := mustParseURL(trial)
 634                 req := &http.Request{
 635                         Method:     "GET",
 636                         Host:       u.Host,
 637                         URL:        u,
 638                         RequestURI: u.RequestURI(),
 639                 }
 640                 resp := httptest.NewRecorder()
 641                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 642                 s.handler.ServeHTTP(resp, req)
 643                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 644         }
 645 }
 646
 647 func mustParseURL(s string) *url.URL {
 648         r, err := url.Parse(s)
 649         if err != nil {
 650                 panic("parse URL: " + s)
 651         }
 652         return r
 653 }
 654
 655 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 656         for _, testURL := range []string{
 657                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 658                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 659         } {
 660                 resp := httptest.NewRecorder()
 661                 u := mustParseURL(testURL)
 662                 req := &http.Request{
 663                         Method:     "GET",
 664                         URL:        u,
 665                         RequestURI: u.RequestURI(),
 666                 }
 667                 s.handler.ServeHTTP(resp, req)
 668                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 669                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 670         }
 671 }
 672
 673 // An authorizer modifies an HTTP request to make use of the given
 674 // token -- by adding it to a header, cookie, query param, or whatever
 675 // -- and returns the HTTP status code we should expect from keep-web if
 676 // the token is invalid.
 677 type authorizer func(*http.Request, string) int
 678
 679 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 680         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 681 }
 682 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 683         r.Header.Add("Authorization", "OAuth2 "+tok)
 684         return http.StatusUnauthorized
 685 }
 686
 687 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 688         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 689 }
 690 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 691         r.Header.Add("Authorization", "Bearer "+tok)
 692         return http.StatusUnauthorized
 693 }
 694
 695 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 696         s.doVhostRequests(c, authzViaCookieValue)
 697 }
 698 func authzViaCookieValue(r *http.Request, tok string) int {
 699         r.AddCookie(&http.Cookie{
 700                 Name:  "arvados_api_token",
 701                 Value: auth.EncodeTokenCookie([]byte(tok)),
 702         })
 703         return http.StatusUnauthorized
 704 }
 705
 706 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 707         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 708 }
 709 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 710         r.AddCookie(&http.Cookie{
 711                 Name:  "arvados_api_token",
 712                 Value: auth.EncodeTokenCookie([]byte(tok)),
 713         })
 714         return http.StatusUnauthorized
 715 }
 716
 717 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 718         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 719                 r.AddCookie(&http.Cookie{
 720                         Name:  "arvados_api_token",
 721                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 722                 })
 723                 return http.StatusUnauthorized
 724         })
 725 }
 726
 727 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 728         s.doVhostRequests(c, authzViaPath)
 729 }
 730 func authzViaPath(r *http.Request, tok string) int {
 731         r.URL.Path = "/t=" + tok + r.URL.Path
 732         return http.StatusNotFound
 733 }
 734
 735 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 736         s.doVhostRequests(c, authzViaQueryString)
 737 }
 738 func authzViaQueryString(r *http.Request, tok string) int {
 739         r.URL.RawQuery = "api_token=" + tok
 740         return http.StatusUnauthorized
 741 }
 742
 743 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 744         s.doVhostRequests(c, authzViaPOST)
 745 }
 746 func authzViaPOST(r *http.Request, tok string) int {
 747         r.Method = "POST"
 748         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 749         r.Body = ioutil.NopCloser(strings.NewReader(
 750                 url.Values{"api_token": {tok}}.Encode()))
 751         return http.StatusUnauthorized
 752 }
 753
 754 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 755         s.doVhostRequests(c, authzViaPOST)
 756 }
 757 func authzViaXHRPOST(r *http.Request, tok string) int {
 758         r.Method = "POST"
 759         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 760         r.Header.Add("Origin", "https://origin.example")
 761         r.Body = ioutil.NopCloser(strings.NewReader(
 762                 url.Values{
 763                         "api_token":   {tok},
 764                         "disposition": {"attachment"},
 765                 }.Encode()))
 766         return http.StatusUnauthorized
 767 }
 768
 769 // Try some combinations of {url, token} using the given authorization
 770 // mechanism, and verify the result is correct.
 771 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 772         for _, hostPath := range []string{
 773                 arvadostest.FooCollection + ".example.com/foo",
 774                 arvadostest.FooCollection + "--collections.example.com/foo",
 775                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 776                 arvadostest.FooCollectionPDH + ".example.com/foo",
 777                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 778                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 779         } {
 780                 c.Log("doRequests: ", hostPath)
 781                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 782         }
 783 }
 784
 785 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 786         for _, tok := range []string{
 787                 arvadostest.ActiveToken,
 788                 arvadostest.ActiveToken[:15],
 789                 arvadostest.SpectatorToken,
 790                 "bogus",
 791                 "",
 792         } {
 793                 u := mustParseURL("http://" + hostPath)
 794                 req := &http.Request{
 795                         Method:     "GET",
 796                         Host:       u.Host,
 797                         URL:        u,
 798                         RequestURI: u.RequestURI(),
 799                         Header:     http.Header{},
 800                 }
 801                 failCode := authz(req, tok)
 802                 req, resp := s.doReq(req)
 803                 code, body := resp.Code, resp.Body.String()
 804
 805                 // If the initial request had a (non-empty) token
 806                 // showing in the query string, we should have been
 807                 // redirected in order to hide it in a cookie.
 808                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 809
 810                 if tok == arvadostest.ActiveToken {
 811                         c.Check(code, check.Equals, http.StatusOK)
 812                         c.Check(body, check.Equals, "foo")
 813                 } else {
 814                         c.Check(code >= 400, check.Equals, true)
 815                         c.Check(code < 500, check.Equals, true)
 816                         if tok == arvadostest.SpectatorToken {
 817                                 // Valid token never offers to retry
 818                                 // with different credentials.
 819                                 c.Check(code, check.Equals, http.StatusNotFound)
 820                         } else {
 821                                 // Invalid token can ask to retry
 822                                 // depending on the authz method.
 823                                 c.Check(code, check.Equals, failCode)
 824                         }
 825                         if code == 404 {
 826                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 827                         } else {
 828                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 829                         }
 830                 }
 831         }
 832 }
 833
 834 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 835         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 836                 for _, port := range []string{"80", "443", "8000"} {
 837                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 838                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 839                         req := &http.Request{
 840                                 Method:     "GET",
 841                                 Host:       u.Host,
 842                                 URL:        u,
 843                                 RequestURI: u.RequestURI(),
 844                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 845                         }
 846                         req, resp := s.doReq(req)
 847                         code, _ := resp.Code, resp.Body.String()
 848
 849                         if port == "8000" {
 850                                 c.Check(code, check.Equals, 401)
 851                         } else {
 852                                 c.Check(code, check.Equals, 200)
 853                         }
 854                 }
 855         }
 856 }
 857
 858 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 859         u := mustParseURL(urlstring)
 860         if hdr == nil && token != "" {
 861                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 862         } else if hdr == nil {
 863                 hdr = http.Header{}
 864         } else if token != "" {
 865                 panic("must not pass both token and hdr")
 866         }
 867         return s.doReq(&http.Request{
 868                 Method:     method,
 869                 Host:       u.Host,
 870                 URL:        u,
 871                 RequestURI: u.RequestURI(),
 872                 Header:     hdr,
 873         })
 874 }
 875
 876 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 877         resp := httptest.NewRecorder()
 878         s.handler.ServeHTTP(resp, req)
 879         if resp.Code != http.StatusSeeOther {
 880                 return req, resp
 881         }
 882         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 883         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 884         req = &http.Request{
 885                 Method:     "GET",
 886                 Host:       u.Host,
 887                 URL:        u,
 888                 RequestURI: u.RequestURI(),
 889                 Header:     http.Header{},
 890         }
 891         for _, c := range cookies {
 892                 req.AddCookie(c)
 893         }
 894         return s.doReq(req)
 895 }
 896
 897 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 898         s.testVhostRedirectTokenToCookie(c, "GET",
 899                 arvadostest.FooCollection+".example.com/foo",
 900                 "?api_token="+arvadostest.ActiveToken,
 901                 nil,
 902                 "",
 903                 http.StatusOK,
 904                 "foo",
 905         )
 906 }
 907
 908 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 909         s.testVhostRedirectTokenToCookie(c, "GET",
 910                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 911                 "",
 912                 nil,
 913                 "",
 914                 http.StatusOK,
 915                 "foo",
 916         )
 917 }
 918
 919 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 920         s.testVhostRedirectTokenToCookie(c, "GET",
 921                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 922                 "",
 923                 nil,
 924                 "",
 925                 http.StatusOK,
 926                 "foo",
 927         )
 928         // Same valid sharing token, but requesting a different collection
 929         s.testVhostRedirectTokenToCookie(c, "GET",
 930                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 931                 "",
 932                 nil,
 933                 "",
 934                 http.StatusNotFound,
 935                 regexp.QuoteMeta(notFoundMessage+"\n"),
 936         )
 937 }
 938
 939 // Bad token in URL is 404 Not Found because it doesn't make sense to
 940 // retry the same URL with different authorization.
 941 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 942         s.testVhostRedirectTokenToCookie(c, "GET",
 943                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 944                 "",
 945                 nil,
 946                 "",
 947                 http.StatusNotFound,
 948                 regexp.QuoteMeta(notFoundMessage+"\n"),
 949         )
 950 }
 951
 952 // Bad token in a cookie (even if it got there via our own
 953 // query-string-to-cookie redirect) is, in principle, retryable via
 954 // wb2-login-and-redirect flow.
 955 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 956         // Inline
 957         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 958                 arvadostest.FooCollection+".example.com/foo",
 959                 "?api_token=thisisabogustoken",
 960                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 961                 "",
 962                 http.StatusSeeOther,
 963                 "",
 964         )
 965         u, err := url.Parse(resp.Header().Get("Location"))
 966         c.Assert(err, check.IsNil)
 967         c.Logf("redirected to %s", u)
 968         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 969         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 970         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 971
 972         // Download/attachment indicated by ?disposition=attachment
 973         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 974                 arvadostest.FooCollection+".example.com/foo",
 975                 "?api_token=thisisabogustoken&disposition=attachment",
 976                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 977                 "",
 978                 http.StatusSeeOther,
 979                 "",
 980         )
 981         u, err = url.Parse(resp.Header().Get("Location"))
 982         c.Assert(err, check.IsNil)
 983         c.Logf("redirected to %s", u)
 984         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 985         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 986         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 987
 988         // Download/attachment indicated by vhost
 989         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 990                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 991                 "?api_token=thisisabogustoken",
 992                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 993                 "",
 994                 http.StatusSeeOther,
 995                 "",
 996         )
 997         u, err = url.Parse(resp.Header().Get("Location"))
 998         c.Assert(err, check.IsNil)
 999         c.Logf("redirected to %s", u)
1000         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1001         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
1002         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1003
1004         // Without "Sec-Fetch-Mode: navigate" header, just 401.
1005         s.testVhostRedirectTokenToCookie(c, "GET",
1006                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1007                 "?api_token=thisisabogustoken",
1008                 http.Header{"Sec-Fetch-Mode": {"cors"}},
1009                 "",
1010                 http.StatusUnauthorized,
1011                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
1012         )
1013         s.testVhostRedirectTokenToCookie(c, "GET",
1014                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1015                 "?api_token=thisisabogustoken",
1016                 nil,
1017                 "",
1018                 http.StatusUnauthorized,
1019                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
1020         )
1021 }
1022
1023 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
1024         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1025                 arvadostest.FooCollection+".example.com/foo",
1026                 "?api_token=thisisabogustoken",
1027                 http.Header{
1028                         "Sec-Fetch-Mode": {"navigate"},
1029                         "Cache-Control":  {"no-cache"},
1030                 },
1031                 "",
1032                 http.StatusSeeOther,
1033                 "",
1034         )
1035         u, err := url.Parse(resp.Header().Get("Location"))
1036         c.Assert(err, check.IsNil)
1037         c.Logf("redirected to %s", u)
1038         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1039         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1040         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
1041 }
1042
1043 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
1044         for _, trial := range []struct {
1045                 anonToken    bool
1046                 cacheControl string
1047         }{
1048                 {},
1049                 {cacheControl: "no-cache"},
1050                 {anonToken: true},
1051                 {anonToken: true, cacheControl: "no-cache"},
1052         } {
1053                 c.Logf("trial: %+v", trial)
1054
1055                 if trial.anonToken {
1056                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1057                 } else {
1058                         s.handler.Cluster.Users.AnonymousUserToken = ""
1059                 }
1060                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1061                 c.Assert(err, check.IsNil)
1062                 req.Header.Set("Sec-Fetch-Mode", "navigate")
1063                 if trial.cacheControl != "" {
1064                         req.Header.Set("Cache-Control", trial.cacheControl)
1065                 }
1066                 resp := httptest.NewRecorder()
1067                 s.handler.ServeHTTP(resp, req)
1068                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
1069                 u, err := url.Parse(resp.Header().Get("Location"))
1070                 c.Assert(err, check.IsNil)
1071                 c.Logf("redirected to %q", u)
1072                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1073                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1074                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
1075         }
1076 }
1077
1078 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
1079         s.testVhostRedirectTokenToCookie(c, "GET",
1080                 "example.com/c="+arvadostest.FooCollection+"/foo",
1081                 "?api_token="+arvadostest.ActiveToken,
1082                 nil,
1083                 "",
1084                 http.StatusBadRequest,
1085                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
1086         )
1087 }
1088
1089 // If client requests an attachment by putting ?disposition=attachment
1090 // in the query string, and gets redirected, the redirect target
1091 // should respond with an attachment.
1092 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
1093         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1094                 arvadostest.FooCollection+".example.com/foo",
1095                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
1096                 nil,
1097                 "",
1098                 http.StatusOK,
1099                 "foo",
1100         )
1101         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1102 }
1103
1104 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
1105         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1106         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1107                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
1108                 "?api_token="+arvadostest.ActiveToken,
1109                 nil,
1110                 "",
1111                 http.StatusOK,
1112                 "foo",
1113         )
1114         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1115 }
1116
1117 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
1118         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1119         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1120                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
1121                 "?api_token="+arvadostest.ActiveToken,
1122                 nil,
1123                 "",
1124                 http.StatusOK,
1125                 "waz",
1126         )
1127         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1128         resp = s.testVhostRedirectTokenToCookie(c, "GET",
1129                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
1130                 "?api_token="+arvadostest.ActiveToken,
1131                 nil,
1132                 "",
1133                 http.StatusOK,
1134                 "waz",
1135         )
1136         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1137 }
1138
1139 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
1140         s.handler.Cluster.Collections.TrustAllContent = true
1141         s.testVhostRedirectTokenToCookie(c, "GET",
1142                 "example.com/c="+arvadostest.FooCollection+"/foo",
1143                 "?api_token="+arvadostest.ActiveToken,
1144                 nil,
1145                 "",
1146                 http.StatusOK,
1147                 "foo",
1148         )
1149 }
1150
1151 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
1152         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
1153
1154         s.testVhostRedirectTokenToCookie(c, "GET",
1155                 "example.com/c="+arvadostest.FooCollection+"/foo",
1156                 "?api_token="+arvadostest.ActiveToken,
1157                 nil,
1158                 "",
1159                 http.StatusBadRequest,
1160                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
1161         )
1162
1163         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1164                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
1165                 "?api_token="+arvadostest.ActiveToken,
1166                 nil,
1167                 "",
1168                 http.StatusOK,
1169                 "foo",
1170         )
1171         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
1172 }
1173
1174 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
1175         baseUrl := arvadostest.FooCollection + ".example.com/foo"
1176         query := url.Values{}
1177
1178         // The intent of these tests is to check that requests are redirected
1179         // correctly in the presence of multiple API tokens. The exact response
1180         // codes and content are not closely considered: they're just how
1181         // keep-web responded when we made the smallest possible fix. Changing
1182         // those responses may be okay, but you should still test all these
1183         // different cases and the associated redirect logic.
1184         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
1185         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1186         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
1187         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1188         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
1189         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1190         query["api_token"] = []string{"", arvadostest.ActiveToken}
1191         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1192
1193         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
1194         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
1195         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1196         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
1197         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1198         query["api_token"] = []string{"", arvadostest.AnonymousToken}
1199         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1200 }
1201
1202 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
1203         s.testVhostRedirectTokenToCookie(c, "POST",
1204                 arvadostest.FooCollection+".example.com/foo",
1205                 "",
1206                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
1207                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
1208                 http.StatusOK,
1209                 "foo",
1210         )
1211 }
1212
1213 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
1214         s.testVhostRedirectTokenToCookie(c, "POST",
1215                 arvadostest.FooCollection+".example.com/foo",
1216                 "",
1217                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
1218                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
1219                 http.StatusNotFound,
1220                 regexp.QuoteMeta(notFoundMessage+"\n"),
1221         )
1222 }
1223
1224 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
1225         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1226         s.testVhostRedirectTokenToCookie(c, "GET",
1227                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
1228                 "",
1229                 nil,
1230                 "",
1231                 http.StatusOK,
1232                 "Hello world\n",
1233         )
1234 }
1235
1236 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
1237         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
1238         s.testVhostRedirectTokenToCookie(c, "GET",
1239                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
1240                 "",
1241                 nil,
1242                 "",
1243                 http.StatusUnauthorized,
1244                 "Authorization tokens are not accepted here: .*\n",
1245         )
1246 }
1247
1248 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
1249         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1250
1251         client := arvados.NewClientFromEnv()
1252         client.AuthToken = arvadostest.ActiveToken
1253         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
1254         c.Assert(err, check.IsNil)
1255         path := `https:\\"odd' path chars`
1256         f, err := fs.OpenFile(path, os.O_CREATE, 0777)
1257         c.Assert(err, check.IsNil)
1258         f.Close()
1259         mtxt, err := fs.MarshalManifest(".")
1260         c.Assert(err, check.IsNil)
1261         var coll arvados.Collection
1262         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1263                 "collection": map[string]string{
1264                         "manifest_text": mtxt,
1265                 },
1266         })
1267         c.Assert(err, check.IsNil)
1268
1269         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
1270         req := &http.Request{
1271                 Method:     "GET",
1272                 Host:       u.Host,
1273                 URL:        u,
1274                 RequestURI: u.RequestURI(),
1275                 Header: http.Header{
1276                         "Authorization": {"Bearer " + client.AuthToken},
1277                 },
1278         }
1279         resp := httptest.NewRecorder()
1280         s.handler.ServeHTTP(resp, req)
1281         c.Check(resp.Code, check.Equals, http.StatusOK)
1282         doc, err := html.Parse(resp.Body)
1283         c.Assert(err, check.IsNil)
1284         pathHrefMap := getPathHrefMap(doc)
1285         c.Check(pathHrefMap, check.HasLen, 1) // the one leaf added to collection
1286         href, hasPath := pathHrefMap[path]
1287         c.Assert(hasPath, check.Equals, true) // the path is listed
1288         relUrl := mustParseURL(href)
1289         c.Check(relUrl.Path, check.Equals, "./"+path) // href can be decoded back to path
1290 }
1291
1292 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
1293         arv := arvados.NewClientFromEnv()
1294         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1295         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
1296         name := "foo/bar/baz"
1297         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
1298
1299         client := arvados.NewClientFromEnv()
1300         client.AuthToken = arvadostest.ActiveToken
1301         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
1302         c.Assert(err, check.IsNil)
1303         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
1304         c.Assert(err, check.IsNil)
1305         f.Close()
1306         mtxt, err := fs.MarshalManifest(".")
1307         c.Assert(err, check.IsNil)
1308         var coll arvados.Collection
1309         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1310                 "collection": map[string]string{
1311                         "manifest_text": mtxt,
1312                         "name":          name,
1313                         "owner_uuid":    arvadostest.AProjectUUID,
1314                 },
1315         })
1316         c.Assert(err, check.IsNil)
1317         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1318
1319         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
1320         for tryURL, expectedAnchorText := range map[string]string{
1321                 base:                   nameShown + "/",
1322                 base + nameShown + "/": "filename",
1323         } {
1324                 u, _ := url.Parse(tryURL)
1325                 req := &http.Request{
1326                         Method:     "GET",
1327                         Host:       u.Host,
1328                         URL:        u,
1329                         RequestURI: u.RequestURI(),
1330                         Header: http.Header{
1331                                 "Authorization": {"Bearer " + client.AuthToken},
1332                         },
1333                 }
1334                 resp := httptest.NewRecorder()
1335                 s.handler.ServeHTTP(resp, req)
1336                 c.Check(resp.Code, check.Equals, http.StatusOK)
1337                 doc, err := html.Parse(resp.Body)
1338                 c.Assert(err, check.IsNil) // valid HTML
1339                 pathHrefMap := getPathHrefMap(doc)
1340                 href, hasExpected := pathHrefMap[expectedAnchorText]
1341                 c.Assert(hasExpected, check.Equals, true) // has expected anchor text
1342                 c.Assert(href, check.Not(check.Equals), "")
1343                 relUrl := mustParseURL(href)
1344                 c.Check(relUrl.Path, check.Equals, "./"+expectedAnchorText) // decoded href maps back to the anchor text
1345         }
1346 }
1347
1348 // XHRs can't follow redirect-with-cookie so they rely on method=POST
1349 // and disposition=attachment (telling us it's acceptable to respond
1350 // with content instead of a redirect) and an Origin header that gets
1351 // added automatically by the browser (telling us it's desirable to do
1352 // so).
1353 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
1354         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
1355         req := &http.Request{
1356                 Method:     "POST",
1357                 Host:       u.Host,
1358                 URL:        u,
1359                 RequestURI: u.RequestURI(),
1360                 Header: http.Header{
1361                         "Origin":       {"https://origin.example"},
1362                         "Content-Type": {"application/x-www-form-urlencoded"},
1363                 },
1364                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1365                         "api_token":   {arvadostest.ActiveToken},
1366                         "disposition": {"attachment"},
1367                 }.Encode())),
1368         }
1369         resp := httptest.NewRecorder()
1370         s.handler.ServeHTTP(resp, req)
1371         c.Check(resp.Code, check.Equals, http.StatusOK)
1372         c.Check(resp.Body.String(), check.Equals, "foo")
1373         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1374
1375         // GET + Origin header is representative of both AJAX GET
1376         // requests and inline images via <IMG crossorigin="anonymous"
1377         // src="...">.
1378         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1379         req = &http.Request{
1380                 Method:     "GET",
1381                 Host:       u.Host,
1382                 URL:        u,
1383                 RequestURI: u.RequestURI(),
1384                 Header: http.Header{
1385                         "Origin": {"https://origin.example"},
1386                 },
1387         }
1388         resp = httptest.NewRecorder()
1389         s.handler.ServeHTTP(resp, req)
1390         c.Check(resp.Code, check.Equals, http.StatusOK)
1391         c.Check(resp.Body.String(), check.Equals, "foo")
1392         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1393 }
1394
1395 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1396         if reqHeader == nil {
1397                 reqHeader = http.Header{}
1398         }
1399         u, _ := url.Parse(`http://` + hostPath + queryString)
1400         c.Logf("requesting %s", u)
1401         req := &http.Request{
1402                 Method:     method,
1403                 Host:       u.Host,
1404                 URL:        u,
1405                 RequestURI: u.RequestURI(),
1406                 Header:     reqHeader,
1407                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1408         }
1409
1410         resp := httptest.NewRecorder()
1411         defer func() {
1412                 c.Check(resp.Code, check.Equals, expectStatus)
1413                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1414         }()
1415
1416         s.handler.ServeHTTP(resp, req)
1417         if resp.Code != http.StatusSeeOther {
1418                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1419                 // Since we're not redirecting, check that any api_token in the URL is
1420                 // handled safely.
1421                 // If there is no token in the URL, then we're good.
1422                 // Otherwise, if the response code is an error, the body is expected to
1423                 // be static content, and nothing that might maliciously introspect the
1424                 // URL. It's considered safe and allowed.
1425                 // Otherwise, if the response content has attachment disposition,
1426                 // that's considered safe for all the reasons explained in the
1427                 // safeAttachment comment in handler.go.
1428                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1429                 return resp
1430         }
1431
1432         loc, err := url.Parse(resp.Header().Get("Location"))
1433         c.Assert(err, check.IsNil)
1434         c.Check(loc.Scheme, check.Equals, u.Scheme)
1435         c.Check(loc.Host, check.Equals, u.Host)
1436         c.Check(loc.RawPath, check.Equals, u.RawPath)
1437         // If the response was a redirect, it should never include an API token.
1438         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1439         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1440         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1441
1442         c.Logf("following redirect to %s", u)
1443         req = &http.Request{
1444                 Method:     "GET",
1445                 Host:       loc.Host,
1446                 URL:        loc,
1447                 RequestURI: loc.RequestURI(),
1448                 Header:     reqHeader,
1449         }
1450         for _, c := range cookies {
1451                 req.AddCookie(c)
1452         }
1453
1454         resp = httptest.NewRecorder()
1455         s.handler.ServeHTTP(resp, req)
1456
1457         if resp.Code != http.StatusSeeOther {
1458                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1459         }
1460         return resp
1461 }
1462
1463 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1464         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1465         s.testDirectoryListing(c)
1466 }
1467
1468 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1469         s.handler.Cluster.Users.AnonymousUserToken = ""
1470         s.testDirectoryListing(c)
1471 }
1472
1473 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1474         // The "ownership cycle" test fixtures are reachable from the
1475         // "filter group without filters" group, causing webdav's
1476         // walkfs to recurse indefinitely. Avoid that by deleting one
1477         // of the bogus fixtures.
1478         arv := arvados.NewClientFromEnv()
1479         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1480         if err != nil {
1481                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1482                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1483         }
1484
1485         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1486         authHeader := http.Header{
1487                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1488         }
1489         for _, trial := range []struct {
1490                 uri      string
1491                 header   http.Header
1492                 expect   []string
1493                 redirect string
1494                 cutDirs  int
1495         }{
1496                 {
1497                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1498                         header:  authHeader,
1499                         expect:  []string{"dir1/foo", "dir1/bar"},
1500                         cutDirs: 0,
1501                 },
1502                 {
1503                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1504                         header:  authHeader,
1505                         expect:  []string{"foo", "bar"},
1506                         cutDirs: 1,
1507                 },
1508                 {
1509                         // URLs of this form ignore authHeader, and
1510                         // FooAndBarFilesInDirUUID isn't public, so
1511                         // this returns 401.
1512                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1513                         header: authHeader,
1514                         expect: nil,
1515                 },
1516                 {
1517                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1518                         header:  authHeader,
1519                         expect:  []string{"dir1/"},
1520                         cutDirs: 3,
1521                 },
1522                 {
1523                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1524                         header:  authHeader,
1525                         expect:  []string{"bar"},
1526                         cutDirs: 4,
1527                 },
1528                 {
1529                         uri:     "download.example.com/",
1530                         header:  authHeader,
1531                         expect:  []string{"users/"},
1532                         cutDirs: 0,
1533                 },
1534                 {
1535                         uri:      "download.example.com/users",
1536                         header:   authHeader,
1537                         redirect: "/users/",
1538                         expect:   []string{"active/"},
1539                         cutDirs:  1,
1540                 },
1541                 {
1542                         uri:     "download.example.com/users/",
1543                         header:  authHeader,
1544                         expect:  []string{"active/"},
1545                         cutDirs: 1,
1546                 },
1547                 {
1548                         uri:      "download.example.com/users/active",
1549                         header:   authHeader,
1550                         redirect: "/users/active/",
1551                         expect:   []string{"foo_file_in_dir/"},
1552                         cutDirs:  2,
1553                 },
1554                 {
1555                         uri:     "download.example.com/users/active/",
1556                         header:  authHeader,
1557                         expect:  []string{"foo_file_in_dir/"},
1558                         cutDirs: 2,
1559                 },
1560                 {
1561                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1562                         header:  nil,
1563                         expect:  []string{"dir1/foo", "dir1/bar"},
1564                         cutDirs: 4,
1565                 },
1566                 {
1567                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1568                         header:  nil,
1569                         expect:  []string{"dir1/foo", "dir1/bar"},
1570                         cutDirs: 2,
1571                 },
1572                 {
1573                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1574                         header:  nil,
1575                         expect:  []string{"dir1/foo", "dir1/bar"},
1576                         cutDirs: 2,
1577                 },
1578                 {
1579                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1580                         header:  authHeader,
1581                         expect:  []string{"dir1/foo", "dir1/bar"},
1582                         cutDirs: 1,
1583                 },
1584                 {
1585                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1586                         header:   authHeader,
1587                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1588                         expect:   []string{"foo", "bar"},
1589                         cutDirs:  2,
1590                 },
1591                 {
1592                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1593                         header:  authHeader,
1594                         expect:  []string{"foo", "bar"},
1595                         cutDirs: 3,
1596                 },
1597                 {
1598                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1599                         header:   authHeader,
1600                         redirect: "/dir1/",
1601                         expect:   []string{"foo", "bar"},
1602                         cutDirs:  1,
1603                 },
1604                 {
1605                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1606                         header: authHeader,
1607                         expect: nil,
1608                 },
1609                 {
1610                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1611                         header:  authHeader,
1612                         expect:  []string{"waz"},
1613                         cutDirs: 1,
1614                 },
1615                 {
1616                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1617                         header:  authHeader,
1618                         expect:  []string{"waz"},
1619                         cutDirs: 2,
1620                 },
1621                 {
1622                         uri:     "download.example.com/users/active/This filter group/",
1623                         header:  authHeader,
1624                         expect:  []string{"A Subproject/"},
1625                         cutDirs: 3,
1626                 },
1627                 {
1628                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1629                         header:  authHeader,
1630                         expect:  []string{"baz_file/"},
1631                         cutDirs: 4,
1632                 },
1633                 {
1634                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1635                         header:  authHeader,
1636                         expect:  []string{"A Subproject/"},
1637                         cutDirs: 2,
1638                 },
1639                 {
1640                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1641                         header:  authHeader,
1642                         expect:  []string{"baz_file/"},
1643                         cutDirs: 3,
1644                 },
1645         } {
1646                 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1647                 resp := httptest.NewRecorder()
1648                 u := mustParseURL("//" + trial.uri)
1649                 req := &http.Request{
1650                         Method:     "GET",
1651                         Host:       u.Host,
1652                         URL:        u,
1653                         RequestURI: u.RequestURI(),
1654                         Header:     copyHeader(trial.header),
1655                 }
1656                 s.handler.ServeHTTP(resp, req)
1657                 var cookies []*http.Cookie
1658                 for resp.Code == http.StatusSeeOther {
1659                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1660                         req = &http.Request{
1661                                 Method:     "GET",
1662                                 Host:       u.Host,
1663                                 URL:        u,
1664                                 RequestURI: u.RequestURI(),
1665                                 Header:     copyHeader(trial.header),
1666                         }
1667                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1668                         for _, c := range cookies {
1669                                 req.AddCookie(c)
1670                         }
1671                         resp = httptest.NewRecorder()
1672                         s.handler.ServeHTTP(resp, req)
1673                 }
1674                 if trial.redirect != "" {
1675                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1676                 }
1677                 if trial.expect == nil {
1678                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1679                 } else {
1680                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1681                         listingPageDoc, err := html.Parse(resp.Body)
1682                         c.Check(err, check.IsNil, comment) // valid HTML document
1683                         pathHrefMap := getPathHrefMap(listingPageDoc)
1684                         c.Assert(pathHrefMap, check.Not(check.HasLen), 0, comment)
1685                         for _, e := range trial.expect {
1686                                 href, hasE := pathHrefMap[e]
1687                                 c.Check(hasE, check.Equals, true, comment) // expected path is listed
1688                                 relUrl := mustParseURL(href)
1689                                 c.Check(relUrl.Path, check.Equals, "./"+e, comment) // href can be decoded back to path
1690                         }
1691                         wgetCommand := getWgetExamplePre(listingPageDoc)
1692                         wgetExpected := regexp.MustCompile(`^\$ wget .*--cut-dirs=(\d+) .*'(https?://[^']+)'$`)
1693                         wgetMatchGroups := wgetExpected.FindStringSubmatch(wgetCommand)
1694                         c.Assert(wgetMatchGroups, check.NotNil)                                     // wget command matches
1695                         c.Check(wgetMatchGroups[1], check.Equals, fmt.Sprintf("%d", trial.cutDirs)) // correct level of cut dirs in wget command
1696                         printedUrl := mustParseURL(wgetMatchGroups[2])
1697                         c.Check(printedUrl.Host, check.Equals, req.URL.Host)
1698                         c.Check(printedUrl.Path, check.Equals, req.URL.Path) // URL arg in wget command can be decoded to the right path
1699                 }
1700
1701                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1702                 req = &http.Request{
1703                         Method:     "OPTIONS",
1704                         Host:       u.Host,
1705                         URL:        u,
1706                         RequestURI: u.RequestURI(),
1707                         Header:     copyHeader(trial.header),
1708                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1709                 }
1710                 resp = httptest.NewRecorder()
1711                 s.handler.ServeHTTP(resp, req)
1712                 if trial.expect == nil {
1713                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1714                 } else {
1715                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1716                 }
1717
1718                 req = &http.Request{
1719                         Method:     "PROPFIND",
1720                         Host:       u.Host,
1721                         URL:        u,
1722                         RequestURI: u.RequestURI(),
1723                         Header:     copyHeader(trial.header),
1724                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1725                 }
1726                 resp = httptest.NewRecorder()
1727                 s.handler.ServeHTTP(resp, req)
1728                 // This check avoids logging a big XML document in the
1729                 // event webdav throws a 500 error after sending
1730                 // headers for a 207.
1731                 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1732                         continue
1733                 }
1734                 if trial.expect == nil {
1735                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1736                 } else {
1737                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1738                         for _, e := range trial.expect {
1739                                 if strings.HasSuffix(e, "/") {
1740                                         e = filepath.Join(u.Path, e) + "/"
1741                                 } else {
1742                                         e = filepath.Join(u.Path, e)
1743                                 }
1744                                 e = strings.Replace(e, " ", "%20", -1)
1745                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1746                         }
1747                 }
1748         }
1749 }
1750
1751 // Shallow-traverse the HTML document, gathering the nodes satisfying the
1752 // predicate function in the output slice. If a node matches the predicate,
1753 // none of its children will be visited.
1754 func getNodes(document *html.Node, predicate func(*html.Node) bool) []*html.Node {
1755         var acc []*html.Node
1756         var traverse func(*html.Node, []*html.Node) []*html.Node
1757         traverse = func(root *html.Node, sofar []*html.Node) []*html.Node {
1758                 if root == nil {
1759                         return sofar
1760                 }
1761                 if predicate(root) {
1762                         return append(sofar, root)
1763                 }
1764                 for cur := root.FirstChild; cur != nil; cur = cur.NextSibling {
1765                         sofar = traverse(cur, sofar)
1766                 }
1767                 return sofar
1768         }
1769         return traverse(document, acc)
1770 }
1771
1772 // Returns true if a node has the attribute targetAttr with the given value
1773 func matchesAttributeValue(node *html.Node, targetAttr string, value string) bool {
1774         for _, attr := range node.Attr {
1775                 if attr.Key == targetAttr && attr.Val == value {
1776                         return true
1777                 }
1778         }
1779         return false
1780 }
1781
1782 // Concatenate the content of text-node children of node; only direct
1783 // children are visited, and any non-text children are skipped.
1784 func getNodeText(node *html.Node) string {
1785         var recv strings.Builder
1786         for c := node.FirstChild; c != nil; c = c.NextSibling {
1787                 if c.Type == html.TextNode {
1788                         recv.WriteString(c.Data)
1789                 }
1790         }
1791         return recv.String()
1792 }
1793
1794 // Returns a map from the directory listing item string (a path) to the href
1795 // value of its <a> tag (an encoded relative URL)
1796 func getPathHrefMap(document *html.Node) map[string]string {
1797         isItemATag := func(node *html.Node) bool {
1798                 return node.Type == html.ElementNode && node.Data == "a" && matchesAttributeValue(node, "class", "item")
1799         }
1800         aTags := getNodes(document, isItemATag)
1801         output := make(map[string]string)
1802         for _, elem := range aTags {
1803                 textContent := getNodeText(elem)
1804                 for _, attr := range elem.Attr {
1805                         if attr.Key == "href" {
1806                                 output[textContent] = attr.Val
1807                                 break
1808                         }
1809                 }
1810         }
1811         return output
1812 }
1813
1814 func getWgetExamplePre(document *html.Node) string {
1815         isWgetPre := func(node *html.Node) bool {
1816                 return node.Type == html.ElementNode && matchesAttributeValue(node, "id", "wget-example")
1817         }
1818         elements := getNodes(document, isWgetPre)
1819         if len(elements) != 1 {
1820                 return ""
1821         }
1822         return getNodeText(elements[0])
1823 }
1824
1825 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1826         arv := arvados.NewClientFromEnv()
1827         var newCollection arvados.Collection
1828         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1829                 "collection": map[string]string{
1830                         "owner_uuid":    arvadostest.ActiveUserUUID,
1831                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1832                         "name":          "keep-web test collection",
1833                 },
1834                 "ensure_unique_name": true,
1835         })
1836         c.Assert(err, check.IsNil)
1837         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1838
1839         var updated arvados.Collection
1840         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1841                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1842                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1843                 req := &http.Request{
1844                         Method:     "DELETE",
1845                         Host:       u.Host,
1846                         URL:        u,
1847                         RequestURI: u.RequestURI(),
1848                         Header: http.Header{
1849                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1850                         },
1851                 }
1852                 resp := httptest.NewRecorder()
1853                 s.handler.ServeHTTP(resp, req)
1854                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1855
1856                 updated = arvados.Collection{}
1857                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1858                 c.Check(err, check.IsNil)
1859                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1860                 c.Logf("updated manifest_text %q", updated.ManifestText)
1861         }
1862         c.Check(updated.ManifestText, check.Equals, "")
1863 }
1864
1865 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1866         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1867
1868         client := arvados.NewClientFromEnv()
1869         client.AuthToken = arvadostest.ActiveToken
1870         arv, err := arvadosclient.New(client)
1871         c.Assert(err, check.Equals, nil)
1872         kc, err := keepclient.MakeKeepClient(arv)
1873         c.Assert(err, check.Equals, nil)
1874
1875         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1876         c.Assert(err, check.IsNil)
1877
1878         trials := []struct {
1879                 filename    string
1880                 content     string
1881                 contentType string
1882         }{
1883                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1884                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1885                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1886                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1887                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1888         }
1889         for _, trial := range trials {
1890                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1891                 c.Assert(err, check.IsNil)
1892                 _, err = f.Write([]byte(trial.content))
1893                 c.Assert(err, check.IsNil)
1894                 c.Assert(f.Close(), check.IsNil)
1895         }
1896         mtxt, err := fs.MarshalManifest(".")
1897         c.Assert(err, check.IsNil)
1898         var coll arvados.Collection
1899         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1900                 "collection": map[string]string{
1901                         "manifest_text": mtxt,
1902                 },
1903         })
1904         c.Assert(err, check.IsNil)
1905
1906         for _, trial := range trials {
1907                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1908                 req := &http.Request{
1909                         Method:     "GET",
1910                         Host:       u.Host,
1911                         URL:        u,
1912                         RequestURI: u.RequestURI(),
1913                         Header: http.Header{
1914                                 "Authorization": {"Bearer " + client.AuthToken},
1915                         },
1916                 }
1917                 resp := httptest.NewRecorder()
1918                 s.handler.ServeHTTP(resp, req)
1919                 c.Check(resp.Code, check.Equals, http.StatusOK)
1920                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1921                 c.Check(resp.Body.String(), check.Equals, trial.content)
1922         }
1923 }
1924
1925 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1926         req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1927         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1928         c.Assert(err, check.IsNil)
1929         resp := httptest.NewRecorder()
1930         s.handler.ServeHTTP(resp, req)
1931         c.Assert(resp.Code, check.Equals, http.StatusOK)
1932         c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1933 }
1934
1935 // Writing to a collection shouldn't affect its entry in the
1936 // PDH-to-manifest cache.
1937 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1938         arv, err := arvadosclient.MakeArvadosClient()
1939         c.Assert(err, check.Equals, nil)
1940         arv.ApiToken = arvadostest.ActiveToken
1941
1942         u := mustParseURL("http://x.example/testfile")
1943         req := &http.Request{
1944                 Method:     "GET",
1945                 Host:       u.Host,
1946                 URL:        u,
1947                 RequestURI: u.RequestURI(),
1948                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1949         }
1950
1951         checkWithID := func(id string, status int) {
1952                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1953                 req.Host = req.URL.Host
1954                 resp := httptest.NewRecorder()
1955                 s.handler.ServeHTTP(resp, req)
1956                 c.Check(resp.Code, check.Equals, status)
1957         }
1958
1959         var colls [2]arvados.Collection
1960         for i := range colls {
1961                 err := arv.Create("collections",
1962                         map[string]interface{}{
1963                                 "ensure_unique_name": true,
1964                                 "collection": map[string]interface{}{
1965                                         "name": "test collection",
1966                                 },
1967                         }, &colls[i])
1968                 c.Assert(err, check.Equals, nil)
1969         }
1970
1971         // Populate cache with empty collection
1972         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1973
1974         // write a file to colls[0]
1975         reqPut := *req
1976         reqPut.Method = "PUT"
1977         reqPut.URL.Host = colls[0].UUID + ".example"
1978         reqPut.Host = req.URL.Host
1979         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1980         resp := httptest.NewRecorder()
1981         s.handler.ServeHTTP(resp, &reqPut)
1982         c.Check(resp.Code, check.Equals, http.StatusCreated)
1983
1984         // new file should not appear in colls[1]
1985         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1986         checkWithID(colls[1].UUID, http.StatusNotFound)
1987
1988         checkWithID(colls[0].UUID, http.StatusOK)
1989 }
1990
1991 func copyHeader(h http.Header) http.Header {
1992         hc := http.Header{}
1993         for k, v := range h {
1994                 hc[k] = append([]string(nil), v...)
1995         }
1996         return hc
1997 }
1998
1999 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
2000         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
2001
2002         client := arvados.NewClientFromEnv()
2003         client.AuthToken = arvadostest.AdminToken
2004         var logentries arvados.LogList
2005         limit1 := 1
2006         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
2007                 arvados.ResourceListParams{
2008                         Limit: &limit1,
2009                         Order: "created_at desc"})
2010         c.Check(err, check.IsNil)
2011         c.Check(logentries.Items, check.HasLen, 1)
2012         lastLogId := logentries.Items[0].ID
2013         c.Logf("lastLogId: %d", lastLogId)
2014
2015         var logbuf bytes.Buffer
2016         logger := logrus.New()
2017         logger.Out = &logbuf
2018         resp := httptest.NewRecorder()
2019         req = req.WithContext(ctxlog.Context(context.Background(), logger))
2020         s.handler.ServeHTTP(resp, req)
2021
2022         if perm {
2023                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
2024                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
2025                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
2026
2027                 deadline := time.Now().Add(time.Second)
2028                 for {
2029                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
2030                         logentries = arvados.LogList{}
2031                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
2032                                 arvados.ResourceListParams{
2033                                         Filters: []arvados.Filter{
2034                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
2035                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
2036                                         },
2037                                         Limit: &limit1,
2038                                         Order: "created_at desc",
2039                                 })
2040                         c.Assert(err, check.IsNil)
2041                         if len(logentries.Items) > 0 &&
2042                                 logentries.Items[0].ID > lastLogId &&
2043                                 logentries.Items[0].ObjectUUID == userUuid &&
2044                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
2045                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
2046                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
2047                                 break
2048                         }
2049                         c.Logf("logentries.Items: %+v", logentries.Items)
2050                         time.Sleep(50 * time.Millisecond)
2051                 }
2052         } else {
2053                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
2054                 c.Check(logbuf.String(), check.Equals, "")
2055         }
2056 }
2057
2058 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
2059         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
2060
2061         s.handler.Cluster.Collections.TrustAllContent = true
2062
2063         for _, adminperm := range []bool{true, false} {
2064                 for _, userperm := range []bool{true, false} {
2065                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
2066                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
2067
2068                         // Test admin permission
2069                         req := &http.Request{
2070                                 Method:     "GET",
2071                                 Host:       u.Host,
2072                                 URL:        u,
2073                                 RequestURI: u.RequestURI(),
2074                                 Header: http.Header{
2075                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
2076                                 },
2077                         }
2078                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
2079                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
2080
2081                         // Test user permission
2082                         req = &http.Request{
2083                                 Method:     "GET",
2084                                 Host:       u.Host,
2085                                 URL:        u,
2086                                 RequestURI: u.RequestURI(),
2087                                 Header: http.Header{
2088                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2089                                 },
2090                         }
2091                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
2092                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
2093                 }
2094         }
2095
2096         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
2097
2098         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
2099                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
2100
2101                 u = mustParseURL(tryurl)
2102                 req := &http.Request{
2103                         Method:     "GET",
2104                         Host:       u.Host,
2105                         URL:        u,
2106                         RequestURI: u.RequestURI(),
2107                         Header: http.Header{
2108                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
2109                         },
2110                 }
2111                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
2112                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
2113         }
2114
2115         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
2116         req := &http.Request{
2117                 Method:     "GET",
2118                 Host:       u.Host,
2119                 URL:        u,
2120                 RequestURI: u.RequestURI(),
2121                 Header: http.Header{
2122                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2123                 },
2124         }
2125         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
2126                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
2127 }
2128
2129 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
2130         for _, adminperm := range []bool{true, false} {
2131                 for _, userperm := range []bool{true, false} {
2132
2133                         arv := arvados.NewClientFromEnv()
2134                         arv.AuthToken = arvadostest.ActiveToken
2135
2136                         var coll arvados.Collection
2137                         err := arv.RequestAndDecode(&coll,
2138                                 "POST",
2139                                 "/arvados/v1/collections",
2140                                 nil,
2141                                 map[string]interface{}{
2142                                         "ensure_unique_name": true,
2143                                         "collection": map[string]interface{}{
2144                                                 "name": "test collection",
2145                                         },
2146                                 })
2147                         c.Assert(err, check.Equals, nil)
2148
2149                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
2150
2151                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
2152                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
2153
2154                         // Test admin permission
2155                         req := &http.Request{
2156                                 Method:     "PUT",
2157                                 Host:       u.Host,
2158                                 URL:        u,
2159                                 RequestURI: u.RequestURI(),
2160                                 Header: http.Header{
2161                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
2162                                 },
2163                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
2164                         }
2165                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
2166                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
2167
2168                         // Test user permission
2169                         req = &http.Request{
2170                                 Method:     "PUT",
2171                                 Host:       u.Host,
2172                                 URL:        u,
2173                                 RequestURI: u.RequestURI(),
2174                                 Header: http.Header{
2175                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2176                                 },
2177                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
2178                         }
2179                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
2180                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
2181                 }
2182         }
2183 }
2184
2185 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
2186         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
2187         lockTidyInterval = time.Second
2188         client := arvados.NewClientFromEnv()
2189         client.AuthToken = arvadostest.ActiveTokenV2
2190         // Start small, and increase concurrency (2^2, 4^2, ...)
2191         // only until hitting failure. Avoids unnecessarily long
2192         // failure reports.
2193         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
2194                 c.Logf("%s: n=%d", c.TestName(), n)
2195
2196                 var coll arvados.Collection
2197                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
2198                 c.Assert(err, check.IsNil)
2199                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
2200
2201                 var wg sync.WaitGroup
2202                 for i := 0; i < n && !c.Failed(); i++ {
2203                         i := i
2204                         wg.Add(1)
2205                         go func() {
2206                                 defer wg.Done()
2207                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
2208                                 resp := httptest.NewRecorder()
2209                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
2210                                 c.Assert(err, check.IsNil)
2211                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2212                                 s.handler.ServeHTTP(resp, req)
2213                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
2214                                 for j := 0; j < n && !c.Failed(); j++ {
2215                                         j := j
2216                                         wg.Add(1)
2217                                         go func() {
2218                                                 defer wg.Done()
2219                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
2220                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
2221
2222                                                 resp := httptest.NewRecorder()
2223                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
2224                                                 c.Assert(err, check.IsNil)
2225                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2226                                                 s.handler.ServeHTTP(resp, req)
2227                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
2228
2229                                                 time.Sleep(time.Second)
2230                                                 resp = httptest.NewRecorder()
2231                                                 req, err = http.NewRequest("GET", u.String(), nil)
2232                                                 c.Assert(err, check.IsNil)
2233                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2234                                                 s.handler.ServeHTTP(resp, req)
2235                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
2236                                                 c.Check(resp.Body.String(), check.Equals, content)
2237                                         }()
2238                                 }
2239                         }()
2240                 }
2241                 wg.Wait()
2242                 for i := 0; i < n; i++ {
2243                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
2244                         resp := httptest.NewRecorder()
2245                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
2246                         c.Assert(err, check.IsNil)
2247                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2248                         s.handler.ServeHTTP(resp, req)
2249                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
2250                 }
2251         }
2252 }