services/keep-web/handler_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net/http"
  14         "net/http/httptest"
  15         "net/url"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20         "sync"
  21         "time"
  22
  23         "git.arvados.org/arvados.git/lib/config"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  26         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  27         "git.arvados.org/arvados.git/sdk/go/auth"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "github.com/prometheus/client_golang/prometheus"
  31         "github.com/sirupsen/logrus"
  32         "golang.org/x/net/html"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var _ = check.Suite(&UnitSuite{})
  37
  38 func init() {
  39         arvados.DebugLocksPanicMode = true
  40 }
  41
  42 type UnitSuite struct {
  43         cluster *arvados.Cluster
  44         handler *handler
  45 }
  46
  47 func (s *UnitSuite) SetUpTest(c *check.C) {
  48         logger := ctxlog.TestLogger(c)
  49         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), logger)
  50         ldr.Path = "-"
  51         cfg, err := ldr.Load()
  52         c.Assert(err, check.IsNil)
  53         cc, err := cfg.GetCluster("")
  54         c.Assert(err, check.IsNil)
  55         s.cluster = cc
  56         s.handler = &handler{
  57                 Cluster: cc,
  58                 Cache: cache{
  59                         cluster:  cc,
  60                         logger:   logger,
  61                         registry: prometheus.NewRegistry(),
  62                 },
  63                 metrics: newMetrics(prometheus.NewRegistry()),
  64         }
  65 }
  66
  67 func newCollection(collID string) *arvados.Collection {
  68         coll := &arvados.Collection{UUID: collID}
  69         if pdh, ok := arvadostest.TestCollectionUUIDToPDH[collID]; ok {
  70                 coll.PortableDataHash = pdh
  71         }
  72         return coll
  73 }
  74
  75 func newRequest(method, urlStr string) *http.Request {
  76         u := mustParseURL(urlStr)
  77         return &http.Request{
  78                 Method:     method,
  79                 Host:       u.Host,
  80                 URL:        u,
  81                 RequestURI: u.RequestURI(),
  82                 RemoteAddr: "10.20.30.40:56789",
  83                 Header:     http.Header{},
  84         }
  85 }
  86
  87 func (s *UnitSuite) TestLogEventTypes(c *check.C) {
  88         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
  89         for method, expected := range map[string]string{
  90                 "GET":  "file_download",
  91                 "POST": "file_upload",
  92                 "PUT":  "file_upload",
  93         } {
  94                 filePath := "/" + method
  95                 req := newRequest(method, collURL+filePath)
  96                 actual := newFileEventLog(s.handler, req, filePath, nil, nil)
  97                 if !c.Check(actual, check.NotNil) {
  98                         continue
  99                 }
 100                 c.Check(actual.eventType, check.Equals, expected)
 101         }
 102 }
 103
 104 func (s *UnitSuite) TestUnloggedEventTypes(c *check.C) {
 105         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 106         for _, method := range []string{"DELETE", "HEAD", "OPTIONS", "PATCH"} {
 107                 filePath := "/" + method
 108                 req := newRequest(method, collURL+filePath)
 109                 actual := newFileEventLog(s.handler, req, filePath, nil, nil)
 110                 c.Check(actual, check.IsNil,
 111                         check.Commentf("%s request made a log event", method))
 112         }
 113 }
 114
 115 func (s *UnitSuite) TestLogFilePath(c *check.C) {
 116         coll := newCollection(arvadostest.FooCollection)
 117         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 118         for _, filePath := range []string{"/foo", "/Foo", "/foo/bar"} {
 119                 req := newRequest("GET", collURL+filePath)
 120                 actual := newFileEventLog(s.handler, req, filePath, coll, nil)
 121                 if !c.Check(actual, check.NotNil) {
 122                         continue
 123                 }
 124                 c.Check(actual.collFilePath, check.Equals, filePath)
 125         }
 126 }
 127
 128 func (s *UnitSuite) TestLogRemoteAddr(c *check.C) {
 129         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 130         filePath := "/foo"
 131         req := newRequest("GET", collURL+filePath)
 132
 133         for _, addr := range []string{"10.20.30.55", "192.168.144.120", "192.0.2.4"} {
 134                 req.RemoteAddr = addr + ":57914"
 135                 actual := newFileEventLog(s.handler, req, filePath, nil, nil)
 136                 if !c.Check(actual, check.NotNil) {
 137                         continue
 138                 }
 139                 c.Check(actual.clientAddr, check.Equals, addr)
 140         }
 141
 142         for _, addr := range []string{"100::20:30:40", "2001:db8::90:100", "3fff::30"} {
 143                 req.RemoteAddr = fmt.Sprintf("[%s]:57916", addr)
 144                 actual := newFileEventLog(s.handler, req, filePath, nil, nil)
 145                 if !c.Check(actual, check.NotNil) {
 146                         continue
 147                 }
 148                 c.Check(actual.clientAddr, check.Equals, addr)
 149         }
 150 }
 151
 152 func (s *UnitSuite) TestLogXForwardedFor(c *check.C) {
 153         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 154         filePath := "/foo"
 155         req := newRequest("GET", collURL+filePath)
 156         for xff, expected := range map[string]string{
 157                 "10.20.30.55":                          "10.20.30.55",
 158                 "192.168.144.120, 10.20.30.120":        "192.168.144.120",
 159                 "192.0.2.4, 192.0.2.6, 192.0.2.8":      "192.0.2.4",
 160                 "192.0.2.4,192.168.2.4":                "192.0.2.4",
 161                 "10.20.30.60,192.168.144.40,192.0.2.4": "10.20.30.60",
 162                 "100::20:30:50":                        "100::20:30:50",
 163                 "2001:db8::80:90, 100::100":            "2001:db8::80:90",
 164                 "3fff::ff, 3fff::ee, 3fff::fe":         "3fff::ff",
 165                 "3fff::3f,100::1000":                   "3fff::3f",
 166                 "2001:db8::88,100::88,3fff::88":        "2001:db8::88",
 167                 "10.20.30.60, 2001:db8::60":            "10.20.30.60",
 168                 "2001:db8::20,10.20.30.20":             "2001:db8::20",
 169                 ", 10.20.30.123, 100::123":             "10.20.30.123",
 170                 ",100::321,10.30.20.10":                "100::321",
 171         } {
 172                 req.Header.Set("X-Forwarded-For", xff)
 173                 actual := newFileEventLog(s.handler, req, filePath, nil, nil)
 174                 if !c.Check(actual, check.NotNil) {
 175                         continue
 176                 }
 177                 c.Check(actual.clientAddr, check.Equals, expected)
 178         }
 179 }
 180
 181 func (s *UnitSuite) TestLogXForwardedForMalformed(c *check.C) {
 182         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 183         filePath := "/foo"
 184         req := newRequest("GET", collURL+filePath)
 185         for _, xff := range []string{"", ",", "10.20,30.40", "foo, bar"} {
 186                 req.Header.Set("X-Forwarded-For", xff)
 187                 actual := newFileEventLog(s.handler, req, filePath, nil, nil)
 188                 if !c.Check(actual, check.NotNil) {
 189                         continue
 190                 }
 191                 c.Check(actual.clientAddr, check.Equals, "10.20.30.40")
 192         }
 193 }
 194
 195 func (s *UnitSuite) TestLogXForwardedForMultivalue(c *check.C) {
 196         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 197         filePath := "/foo"
 198         req := newRequest("GET", collURL+filePath)
 199         req.Header.Set("X-Forwarded-For", ", ")
 200         req.Header.Add("X-Forwarded-For", "2001:db8::db9:dbd")
 201         req.Header.Add("X-Forwarded-For", "10.20.30.90")
 202         actual := newFileEventLog(s.handler, req, filePath, nil, nil)
 203         c.Assert(actual, check.NotNil)
 204         c.Check(actual.clientAddr, check.Equals, "2001:db8::db9:dbd")
 205 }
 206
 207 func (s *UnitSuite) TestLogClientAddressCanonicalization(c *check.C) {
 208         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 209         filePath := "/foo"
 210         req := newRequest("GET", collURL+filePath)
 211         expected := "2001:db8::12:0"
 212
 213         req.RemoteAddr = "[2001:db8::012:0000]:57918"
 214         a := newFileEventLog(s.handler, req, filePath, nil, nil)
 215         c.Assert(a, check.NotNil)
 216         c.Check(a.clientAddr, check.Equals, expected)
 217
 218         req.RemoteAddr = "10.20.30.40:57919"
 219         req.Header.Set("X-Forwarded-For", "2001:db8:0::0:12:00")
 220         b := newFileEventLog(s.handler, req, filePath, nil, nil)
 221         c.Assert(b, check.NotNil)
 222         c.Check(b.clientAddr, check.Equals, expected)
 223 }
 224
 225 func (s *UnitSuite) TestLogAnonymousUser(c *check.C) {
 226         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 227         filePath := "/foo"
 228         req := newRequest("GET", collURL+filePath)
 229         actual := newFileEventLog(s.handler, req, filePath, nil, nil)
 230         c.Assert(actual, check.NotNil)
 231         c.Check(actual.userUUID, check.Equals, s.handler.Cluster.ClusterID+"-tpzed-anonymouspublic")
 232         c.Check(actual.userFullName, check.Equals, "")
 233 }
 234
 235 func (s *UnitSuite) TestLogUser(c *check.C) {
 236         collURL := "http://keep-web.example/c=" + arvadostest.FooCollection
 237         for userUUID, userFullName := range map[string]string{
 238                 arvadostest.ActiveUserUUID:    "Active User",
 239                 arvadostest.SpectatorUserUUID: "Spectator User",
 240         } {
 241                 filePath := "/" + userUUID
 242                 req := newRequest("GET", collURL+filePath)
 243                 user := &arvados.User{
 244                         UUID:     userUUID,
 245                         FullName: userFullName,
 246                 }
 247                 actual := newFileEventLog(s.handler, req, filePath, nil, user)
 248                 if !c.Check(actual, check.NotNil) {
 249                         continue
 250                 }
 251                 c.Check(actual.userUUID, check.Equals, userUUID)
 252                 c.Check(actual.userFullName, check.Equals, userFullName)
 253         }
 254 }
 255
 256 func (s *UnitSuite) TestLogCollectionByUUID(c *check.C) {
 257         for collUUID, collPDH := range arvadostest.TestCollectionUUIDToPDH {
 258                 collURL := "http://keep-web.example/c=" + collUUID
 259                 filePath := "/" + collUUID
 260                 req := newRequest("GET", collURL+filePath)
 261                 coll := newCollection(collUUID)
 262                 actual := newFileEventLog(s.handler, req, filePath, coll, nil)
 263                 if !c.Check(actual, check.NotNil) {
 264                         continue
 265                 }
 266                 c.Check(actual.collUUID, check.Equals, collUUID)
 267                 c.Check(actual.collPDH, check.Equals, collPDH)
 268         }
 269 }
 270
 271 func (s *UnitSuite) TestLogCollectionByPDH(c *check.C) {
 272         for _, collPDH := range arvadostest.TestCollectionUUIDToPDH {
 273                 collURL := "http://keep-web.example/c=" + collPDH
 274                 filePath := "/PDHFile"
 275                 req := newRequest("GET", collURL+filePath)
 276                 coll := newCollection(collPDH)
 277                 actual := newFileEventLog(s.handler, req, filePath, coll, nil)
 278                 if !c.Check(actual, check.NotNil) {
 279                         continue
 280                 }
 281                 c.Check(actual.collPDH, check.Equals, collPDH)
 282                 c.Check(actual.collUUID, check.Equals, "")
 283         }
 284 }
 285
 286 func (s *UnitSuite) TestLogGETUUIDAsDict(c *check.C) {
 287         filePath := "/foo"
 288         reqPath := "/c=" + arvadostest.FooCollection + filePath
 289         req := newRequest("GET", "http://keep-web.example"+reqPath)
 290         coll := newCollection(arvadostest.FooCollection)
 291         logEvent := newFileEventLog(s.handler, req, filePath, coll, nil)
 292         c.Assert(logEvent, check.NotNil)
 293         c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 294                 "event_type":  "file_download",
 295                 "object_uuid": s.handler.Cluster.ClusterID + "-tpzed-anonymouspublic",
 296                 "properties": arvadosclient.Dict{
 297                         "reqPath":              reqPath,
 298                         "collection_uuid":      arvadostest.FooCollection,
 299                         "collection_file_path": filePath,
 300                         "portable_data_hash":   arvadostest.FooCollectionPDH,
 301                 },
 302         })
 303 }
 304
 305 func (s *UnitSuite) TestLogGETPDHAsDict(c *check.C) {
 306         filePath := "/Foo"
 307         reqPath := "/c=" + arvadostest.FooCollectionPDH + filePath
 308         req := newRequest("GET", "http://keep-web.example"+reqPath)
 309         coll := newCollection(arvadostest.FooCollectionPDH)
 310         user := &arvados.User{
 311                 UUID:     arvadostest.ActiveUserUUID,
 312                 FullName: "Active User",
 313         }
 314         logEvent := newFileEventLog(s.handler, req, filePath, coll, user)
 315         c.Assert(logEvent, check.NotNil)
 316         c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 317                 "event_type":  "file_download",
 318                 "object_uuid": arvadostest.ActiveUserUUID,
 319                 "properties": arvadosclient.Dict{
 320                         "reqPath":              reqPath,
 321                         "portable_data_hash":   arvadostest.FooCollectionPDH,
 322                         "collection_uuid":      "",
 323                         "collection_file_path": filePath,
 324                 },
 325         })
 326 }
 327
 328 func (s *UnitSuite) TestLogUploadAsDict(c *check.C) {
 329         coll := newCollection(arvadostest.FooCollection)
 330         user := &arvados.User{
 331                 UUID:     arvadostest.ActiveUserUUID,
 332                 FullName: "Active User",
 333         }
 334         for _, method := range []string{"POST", "PUT"} {
 335                 filePath := "/" + method + "File"
 336                 reqPath := "/c=" + arvadostest.FooCollection + filePath
 337                 req := newRequest(method, "http://keep-web.example"+reqPath)
 338                 logEvent := newFileEventLog(s.handler, req, filePath, coll, user)
 339                 if !c.Check(logEvent, check.NotNil) {
 340                         continue
 341                 }
 342                 c.Check(logEvent.asDict(), check.DeepEquals, arvadosclient.Dict{
 343                         "event_type":  "file_upload",
 344                         "object_uuid": arvadostest.ActiveUserUUID,
 345                         "properties": arvadosclient.Dict{
 346                                 "reqPath":              reqPath,
 347                                 "collection_uuid":      arvadostest.FooCollection,
 348                                 "collection_file_path": filePath,
 349                         },
 350                 })
 351         }
 352 }
 353
 354 func (s *UnitSuite) TestLogGETUUIDAsFields(c *check.C) {
 355         filePath := "/foo"
 356         reqPath := "/c=" + arvadostest.FooCollection + filePath
 357         req := newRequest("GET", "http://keep-web.example"+reqPath)
 358         coll := newCollection(arvadostest.FooCollection)
 359         logEvent := newFileEventLog(s.handler, req, filePath, coll, nil)
 360         c.Assert(logEvent, check.NotNil)
 361         c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 362                 "user_uuid":            s.handler.Cluster.ClusterID + "-tpzed-anonymouspublic",
 363                 "collection_uuid":      arvadostest.FooCollection,
 364                 "collection_file_path": filePath,
 365                 "portable_data_hash":   arvadostest.FooCollectionPDH,
 366         })
 367 }
 368
 369 func (s *UnitSuite) TestLogGETPDHAsFields(c *check.C) {
 370         filePath := "/Foo"
 371         reqPath := "/c=" + arvadostest.FooCollectionPDH + filePath
 372         req := newRequest("GET", "http://keep-web.example"+reqPath)
 373         coll := newCollection(arvadostest.FooCollectionPDH)
 374         user := &arvados.User{
 375                 UUID:     arvadostest.ActiveUserUUID,
 376                 FullName: "Active User",
 377         }
 378         logEvent := newFileEventLog(s.handler, req, filePath, coll, user)
 379         c.Assert(logEvent, check.NotNil)
 380         c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 381                 "user_uuid":            arvadostest.ActiveUserUUID,
 382                 "user_full_name":       "Active User",
 383                 "collection_uuid":      "",
 384                 "collection_file_path": filePath,
 385                 "portable_data_hash":   arvadostest.FooCollectionPDH,
 386         })
 387 }
 388
 389 func (s *UnitSuite) TestLogUploadAsFields(c *check.C) {
 390         coll := newCollection(arvadostest.FooCollection)
 391         user := &arvados.User{
 392                 UUID:     arvadostest.ActiveUserUUID,
 393                 FullName: "Active User",
 394         }
 395         for _, method := range []string{"POST", "PUT"} {
 396                 filePath := "/" + method + "File"
 397                 reqPath := "/c=" + arvadostest.FooCollection + filePath
 398                 req := newRequest(method, "http://keep-web.example"+reqPath)
 399                 logEvent := newFileEventLog(s.handler, req, filePath, coll, user)
 400                 if !c.Check(logEvent, check.NotNil) {
 401                         continue
 402                 }
 403                 c.Check(logEvent.asFields(), check.DeepEquals, logrus.Fields{
 404                         "user_uuid":            arvadostest.ActiveUserUUID,
 405                         "user_full_name":       "Active User",
 406                         "collection_uuid":      arvadostest.FooCollection,
 407                         "collection_file_path": filePath,
 408                 })
 409         }
 410 }
 411
 412 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
 413         h := s.handler
 414         u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
 415         req := &http.Request{
 416                 Method:     "OPTIONS",
 417                 Host:       u.Host,
 418                 URL:        u,
 419                 RequestURI: u.RequestURI(),
 420                 Header: http.Header{
 421                         "Origin":                        {"https://workbench.example"},
 422                         "Access-Control-Request-Method": {"POST"},
 423                 },
 424         }
 425
 426         // Check preflight for an allowed request
 427         resp := httptest.NewRecorder()
 428         h.ServeHTTP(resp, req)
 429         c.Check(resp.Code, check.Equals, http.StatusOK)
 430         c.Check(resp.Body.String(), check.Equals, "")
 431         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
 432         c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Equals, "COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK")
 433         c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Equals, "Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control")
 434
 435         // Check preflight for a disallowed request
 436         resp = httptest.NewRecorder()
 437         req.Header.Set("Access-Control-Request-Method", "MAKE-COFFEE")
 438         h.ServeHTTP(resp, req)
 439         c.Check(resp.Body.String(), check.Equals, "")
 440         c.Check(resp.Code, check.Equals, http.StatusMethodNotAllowed)
 441 }
 442
 443 func (s *UnitSuite) TestWebdavPrefixAndSource(c *check.C) {
 444         for _, trial := range []struct {
 445                 method   string
 446                 path     string
 447                 prefix   string
 448                 source   string
 449                 notFound bool
 450                 seeOther bool
 451         }{
 452                 {
 453                         method: "PROPFIND",
 454                         path:   "/",
 455                 },
 456                 {
 457                         method: "PROPFIND",
 458                         path:   "/dir1",
 459                 },
 460                 {
 461                         method: "PROPFIND",
 462                         path:   "/dir1/",
 463                 },
 464                 {
 465                         method: "PROPFIND",
 466                         path:   "/dir1/foo",
 467                         prefix: "/dir1",
 468                         source: "/dir1",
 469                 },
 470                 {
 471                         method: "PROPFIND",
 472                         path:   "/prefix/dir1/foo",
 473                         prefix: "/prefix/",
 474                         source: "",
 475                 },
 476                 {
 477                         method: "PROPFIND",
 478                         path:   "/prefix/dir1/foo",
 479                         prefix: "/prefix",
 480                         source: "",
 481                 },
 482                 {
 483                         method: "PROPFIND",
 484                         path:   "/prefix/dir1/foo",
 485                         prefix: "/prefix/",
 486                         source: "/",
 487                 },
 488                 {
 489                         method: "PROPFIND",
 490                         path:   "/prefix/foo",
 491                         prefix: "/prefix/",
 492                         source: "/dir1/",
 493                 },
 494                 {
 495                         method: "GET",
 496                         path:   "/prefix/foo",
 497                         prefix: "/prefix/",
 498                         source: "/dir1/",
 499                 },
 500                 {
 501                         method: "PROPFIND",
 502                         path:   "/prefix/",
 503                         prefix: "/prefix",
 504                         source: "/dir1",
 505                 },
 506                 {
 507                         method: "PROPFIND",
 508                         path:   "/prefix",
 509                         prefix: "/prefix",
 510                         source: "/dir1/",
 511                 },
 512                 {
 513                         method:   "GET",
 514                         path:     "/prefix",
 515                         prefix:   "/prefix",
 516                         source:   "/dir1",
 517                         seeOther: true,
 518                 },
 519                 {
 520                         method:   "PROPFIND",
 521                         path:     "/dir1/foo",
 522                         prefix:   "",
 523                         source:   "/dir1",
 524                         notFound: true,
 525                 },
 526         } {
 527                 c.Logf("trial %+v", trial)
 528                 u := mustParseURL("http://" + arvadostest.FooBarDirCollection + ".keep-web.example" + trial.path)
 529                 req := &http.Request{
 530                         Method:     trial.method,
 531                         Host:       u.Host,
 532                         URL:        u,
 533                         RequestURI: u.RequestURI(),
 534                         Header: http.Header{
 535                                 "Authorization":   {"Bearer " + arvadostest.ActiveTokenV2},
 536                                 "X-Webdav-Prefix": {trial.prefix},
 537                                 "X-Webdav-Source": {trial.source},
 538                         },
 539                         Body: ioutil.NopCloser(bytes.NewReader(nil)),
 540                 }
 541
 542                 resp := httptest.NewRecorder()
 543                 s.handler.ServeHTTP(resp, req)
 544                 if trial.notFound {
 545                         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 546                 } else if trial.method == "PROPFIND" {
 547                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
 548                         c.Check(resp.Body.String(), check.Matches, `(?ms).*>\n?$`)
 549                 } else if trial.seeOther {
 550                         c.Check(resp.Code, check.Equals, http.StatusSeeOther)
 551                 } else {
 552                         c.Check(resp.Code, check.Equals, http.StatusOK)
 553                 }
 554         }
 555 }
 556
 557 func (s *UnitSuite) TestEmptyResponse(c *check.C) {
 558         // Ensure we start with an empty cache
 559         defer os.Setenv("HOME", os.Getenv("HOME"))
 560         os.Setenv("HOME", c.MkDir())
 561
 562         for _, trial := range []struct {
 563                 dataExists    bool
 564                 sendIMSHeader bool
 565                 expectStatus  int
 566                 logRegexp     string
 567         }{
 568                 // If we return no content due to a Keep read error,
 569                 // we should emit a log message.
 570                 {false, false, http.StatusOK, `(?ms).*only wrote 0 bytes.*`},
 571
 572                 // If we return no content because the client sent an
 573                 // If-Modified-Since header, our response should be
 574                 // 304.  We still expect a "File download" log since it
 575                 // counts as a file access for auditing.
 576                 {true, true, http.StatusNotModified, `(?ms).*msg="File download".*`},
 577         } {
 578                 c.Logf("trial: %+v", trial)
 579                 arvadostest.StartKeep(2, true)
 580                 if trial.dataExists {
 581                         arv, err := arvadosclient.MakeArvadosClient()
 582                         c.Assert(err, check.IsNil)
 583                         arv.ApiToken = arvadostest.ActiveToken
 584                         kc, err := keepclient.MakeKeepClient(arv)
 585                         c.Assert(err, check.IsNil)
 586                         _, _, err = kc.PutB([]byte("foo"))
 587                         c.Assert(err, check.IsNil)
 588                 }
 589
 590                 u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
 591                 req := &http.Request{
 592                         Method:     "GET",
 593                         Host:       u.Host,
 594                         URL:        u,
 595                         RequestURI: u.RequestURI(),
 596                         Header: http.Header{
 597                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 598                         },
 599                 }
 600                 if trial.sendIMSHeader {
 601                         req.Header.Set("If-Modified-Since", strings.Replace(time.Now().UTC().Format(time.RFC1123), "UTC", "GMT", -1))
 602                 }
 603
 604                 var logbuf bytes.Buffer
 605                 logger := logrus.New()
 606                 logger.Out = &logbuf
 607                 req = req.WithContext(ctxlog.Context(context.Background(), logger))
 608
 609                 resp := httptest.NewRecorder()
 610                 s.handler.ServeHTTP(resp, req)
 611                 c.Check(resp.Code, check.Equals, trial.expectStatus)
 612                 c.Check(resp.Body.String(), check.Equals, "")
 613
 614                 c.Log(logbuf.String())
 615                 c.Check(logbuf.String(), check.Matches, trial.logRegexp)
 616         }
 617 }
 618
 619 func (s *UnitSuite) TestInvalidUUID(c *check.C) {
 620         bogusID := strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "-"
 621         token := arvadostest.ActiveToken
 622         for _, trial := range []string{
 623                 "http://keep-web/c=" + bogusID + "/foo",
 624                 "http://keep-web/c=" + bogusID + "/t=" + token + "/foo",
 625                 "http://keep-web/collections/download/" + bogusID + "/" + token + "/foo",
 626                 "http://keep-web/collections/" + bogusID + "/foo",
 627                 "http://" + bogusID + ".keep-web/" + bogusID + "/foo",
 628                 "http://" + bogusID + ".keep-web/t=" + token + "/" + bogusID + "/foo",
 629         } {
 630                 c.Log(trial)
 631                 u := mustParseURL(trial)
 632                 req := &http.Request{
 633                         Method:     "GET",
 634                         Host:       u.Host,
 635                         URL:        u,
 636                         RequestURI: u.RequestURI(),
 637                 }
 638                 resp := httptest.NewRecorder()
 639                 s.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 640                 s.handler.ServeHTTP(resp, req)
 641                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 642         }
 643 }
 644
 645 func mustParseURL(s string) *url.URL {
 646         r, err := url.Parse(s)
 647         if err != nil {
 648                 panic("parse URL: " + s)
 649         }
 650         return r
 651 }
 652
 653 func (s *IntegrationSuite) TestVhost404(c *check.C) {
 654         for _, testURL := range []string{
 655                 arvadostest.NonexistentCollection + ".example.com/theperthcountyconspiracy",
 656                 arvadostest.NonexistentCollection + ".example.com/t=" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 657         } {
 658                 resp := httptest.NewRecorder()
 659                 u := mustParseURL(testURL)
 660                 req := &http.Request{
 661                         Method:     "GET",
 662                         URL:        u,
 663                         RequestURI: u.RequestURI(),
 664                 }
 665                 s.handler.ServeHTTP(resp, req)
 666                 c.Check(resp.Code, check.Equals, http.StatusNotFound)
 667                 c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
 668         }
 669 }
 670
 671 // An authorizer modifies an HTTP request to make use of the given
 672 // token -- by adding it to a header, cookie, query param, or whatever
 673 // -- and returns the HTTP status code we should expect from keep-web if
 674 // the token is invalid.
 675 type authorizer func(*http.Request, string) int
 676
 677 func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
 678         s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
 679 }
 680 func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
 681         r.Header.Add("Authorization", "OAuth2 "+tok)
 682         return http.StatusUnauthorized
 683 }
 684
 685 func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
 686         s.doVhostRequests(c, authzViaAuthzHeaderBearer)
 687 }
 688 func authzViaAuthzHeaderBearer(r *http.Request, tok string) int {
 689         r.Header.Add("Authorization", "Bearer "+tok)
 690         return http.StatusUnauthorized
 691 }
 692
 693 func (s *IntegrationSuite) TestVhostViaCookieValue(c *check.C) {
 694         s.doVhostRequests(c, authzViaCookieValue)
 695 }
 696 func authzViaCookieValue(r *http.Request, tok string) int {
 697         r.AddCookie(&http.Cookie{
 698                 Name:  "arvados_api_token",
 699                 Value: auth.EncodeTokenCookie([]byte(tok)),
 700         })
 701         return http.StatusUnauthorized
 702 }
 703
 704 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
 705         s.doVhostRequests(c, authzViaHTTPBasicAuth)
 706 }
 707 func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
 708         r.AddCookie(&http.Cookie{
 709                 Name:  "arvados_api_token",
 710                 Value: auth.EncodeTokenCookie([]byte(tok)),
 711         })
 712         return http.StatusUnauthorized
 713 }
 714
 715 func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
 716         s.doVhostRequests(c, func(r *http.Request, tok string) int {
 717                 r.AddCookie(&http.Cookie{
 718                         Name:  "arvados_api_token",
 719                         Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
 720                 })
 721                 return http.StatusUnauthorized
 722         })
 723 }
 724
 725 func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
 726         s.doVhostRequests(c, authzViaPath)
 727 }
 728 func authzViaPath(r *http.Request, tok string) int {
 729         r.URL.Path = "/t=" + tok + r.URL.Path
 730         return http.StatusNotFound
 731 }
 732
 733 func (s *IntegrationSuite) TestVhostViaQueryString(c *check.C) {
 734         s.doVhostRequests(c, authzViaQueryString)
 735 }
 736 func authzViaQueryString(r *http.Request, tok string) int {
 737         r.URL.RawQuery = "api_token=" + tok
 738         return http.StatusUnauthorized
 739 }
 740
 741 func (s *IntegrationSuite) TestVhostViaPOST(c *check.C) {
 742         s.doVhostRequests(c, authzViaPOST)
 743 }
 744 func authzViaPOST(r *http.Request, tok string) int {
 745         r.Method = "POST"
 746         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 747         r.Body = ioutil.NopCloser(strings.NewReader(
 748                 url.Values{"api_token": {tok}}.Encode()))
 749         return http.StatusUnauthorized
 750 }
 751
 752 func (s *IntegrationSuite) TestVhostViaXHRPOST(c *check.C) {
 753         s.doVhostRequests(c, authzViaPOST)
 754 }
 755 func authzViaXHRPOST(r *http.Request, tok string) int {
 756         r.Method = "POST"
 757         r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
 758         r.Header.Add("Origin", "https://origin.example")
 759         r.Body = ioutil.NopCloser(strings.NewReader(
 760                 url.Values{
 761                         "api_token":   {tok},
 762                         "disposition": {"attachment"},
 763                 }.Encode()))
 764         return http.StatusUnauthorized
 765 }
 766
 767 // Try some combinations of {url, token} using the given authorization
 768 // mechanism, and verify the result is correct.
 769 func (s *IntegrationSuite) doVhostRequests(c *check.C, authz authorizer) {
 770         for _, hostPath := range []string{
 771                 arvadostest.FooCollection + ".example.com/foo",
 772                 arvadostest.FooCollection + "--collections.example.com/foo",
 773                 arvadostest.FooCollection + "--collections.example.com/_/foo",
 774                 arvadostest.FooCollectionPDH + ".example.com/foo",
 775                 strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + "--collections.example.com/foo",
 776                 arvadostest.FooBarDirCollection + ".example.com/dir1/foo",
 777         } {
 778                 c.Log("doRequests: ", hostPath)
 779                 s.doVhostRequestsWithHostPath(c, authz, hostPath)
 780         }
 781 }
 782
 783 func (s *IntegrationSuite) doVhostRequestsWithHostPath(c *check.C, authz authorizer, hostPath string) {
 784         for _, tok := range []string{
 785                 arvadostest.ActiveToken,
 786                 arvadostest.ActiveToken[:15],
 787                 arvadostest.SpectatorToken,
 788                 "bogus",
 789                 "",
 790         } {
 791                 u := mustParseURL("http://" + hostPath)
 792                 req := &http.Request{
 793                         Method:     "GET",
 794                         Host:       u.Host,
 795                         URL:        u,
 796                         RequestURI: u.RequestURI(),
 797                         Header:     http.Header{},
 798                 }
 799                 failCode := authz(req, tok)
 800                 req, resp := s.doReq(req)
 801                 code, body := resp.Code, resp.Body.String()
 802
 803                 // If the initial request had a (non-empty) token
 804                 // showing in the query string, we should have been
 805                 // redirected in order to hide it in a cookie.
 806                 c.Check(req.URL.String(), check.Not(check.Matches), `.*api_token=.+`)
 807
 808                 if tok == arvadostest.ActiveToken {
 809                         c.Check(code, check.Equals, http.StatusOK)
 810                         c.Check(body, check.Equals, "foo")
 811                 } else {
 812                         c.Check(code >= 400, check.Equals, true)
 813                         c.Check(code < 500, check.Equals, true)
 814                         if tok == arvadostest.SpectatorToken {
 815                                 // Valid token never offers to retry
 816                                 // with different credentials.
 817                                 c.Check(code, check.Equals, http.StatusNotFound)
 818                         } else {
 819                                 // Invalid token can ask to retry
 820                                 // depending on the authz method.
 821                                 c.Check(code, check.Equals, failCode)
 822                         }
 823                         if code == 404 {
 824                                 c.Check(body, check.Equals, notFoundMessage+"\n")
 825                         } else {
 826                                 c.Check(body, check.Equals, unauthorizedMessage+"\n")
 827                         }
 828                 }
 829         }
 830 }
 831
 832 func (s *IntegrationSuite) TestVhostPortMatch(c *check.C) {
 833         for _, host := range []string{"download.example.com", "DOWNLOAD.EXAMPLE.COM"} {
 834                 for _, port := range []string{"80", "443", "8000"} {
 835                         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = fmt.Sprintf("download.example.com:%v", port)
 836                         u := mustParseURL(fmt.Sprintf("http://%v/by_id/%v/foo", host, arvadostest.FooCollection))
 837                         req := &http.Request{
 838                                 Method:     "GET",
 839                                 Host:       u.Host,
 840                                 URL:        u,
 841                                 RequestURI: u.RequestURI(),
 842                                 Header:     http.Header{"Authorization": []string{"Bearer " + arvadostest.ActiveToken}},
 843                         }
 844                         req, resp := s.doReq(req)
 845                         code, _ := resp.Code, resp.Body.String()
 846
 847                         if port == "8000" {
 848                                 c.Check(code, check.Equals, 401)
 849                         } else {
 850                                 c.Check(code, check.Equals, 200)
 851                         }
 852                 }
 853         }
 854 }
 855
 856 func (s *IntegrationSuite) do(method string, urlstring string, token string, hdr http.Header) (*http.Request, *httptest.ResponseRecorder) {
 857         u := mustParseURL(urlstring)
 858         if hdr == nil && token != "" {
 859                 hdr = http.Header{"Authorization": {"Bearer " + token}}
 860         } else if hdr == nil {
 861                 hdr = http.Header{}
 862         } else if token != "" {
 863                 panic("must not pass both token and hdr")
 864         }
 865         return s.doReq(&http.Request{
 866                 Method:     method,
 867                 Host:       u.Host,
 868                 URL:        u,
 869                 RequestURI: u.RequestURI(),
 870                 Header:     hdr,
 871         })
 872 }
 873
 874 func (s *IntegrationSuite) doReq(req *http.Request) (*http.Request, *httptest.ResponseRecorder) {
 875         resp := httptest.NewRecorder()
 876         s.handler.ServeHTTP(resp, req)
 877         if resp.Code != http.StatusSeeOther {
 878                 return req, resp
 879         }
 880         cookies := (&http.Response{Header: resp.Header()}).Cookies()
 881         u, _ := req.URL.Parse(resp.Header().Get("Location"))
 882         req = &http.Request{
 883                 Method:     "GET",
 884                 Host:       u.Host,
 885                 URL:        u,
 886                 RequestURI: u.RequestURI(),
 887                 Header:     http.Header{},
 888         }
 889         for _, c := range cookies {
 890                 req.AddCookie(c)
 891         }
 892         return s.doReq(req)
 893 }
 894
 895 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToCookie(c *check.C) {
 896         s.testVhostRedirectTokenToCookie(c, "GET",
 897                 arvadostest.FooCollection+".example.com/foo",
 898                 "?api_token="+arvadostest.ActiveToken,
 899                 nil,
 900                 "",
 901                 http.StatusOK,
 902                 "foo",
 903         )
 904 }
 905
 906 func (s *IntegrationSuite) TestSingleOriginSecretLink(c *check.C) {
 907         s.testVhostRedirectTokenToCookie(c, "GET",
 908                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.ActiveToken+"/foo",
 909                 "",
 910                 nil,
 911                 "",
 912                 http.StatusOK,
 913                 "foo",
 914         )
 915 }
 916
 917 func (s *IntegrationSuite) TestCollectionSharingToken(c *check.C) {
 918         s.testVhostRedirectTokenToCookie(c, "GET",
 919                 "example.com/c="+arvadostest.FooFileCollectionUUID+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 920                 "",
 921                 nil,
 922                 "",
 923                 http.StatusOK,
 924                 "foo",
 925         )
 926         // Same valid sharing token, but requesting a different collection
 927         s.testVhostRedirectTokenToCookie(c, "GET",
 928                 "example.com/c="+arvadostest.FooCollection+"/t="+arvadostest.FooFileCollectionSharingToken+"/foo",
 929                 "",
 930                 nil,
 931                 "",
 932                 http.StatusNotFound,
 933                 regexp.QuoteMeta(notFoundMessage+"\n"),
 934         )
 935 }
 936
 937 // Bad token in URL is 404 Not Found because it doesn't make sense to
 938 // retry the same URL with different authorization.
 939 func (s *IntegrationSuite) TestSingleOriginSecretLinkBadToken(c *check.C) {
 940         s.testVhostRedirectTokenToCookie(c, "GET",
 941                 "example.com/c="+arvadostest.FooCollection+"/t=bogus/foo",
 942                 "",
 943                 nil,
 944                 "",
 945                 http.StatusNotFound,
 946                 regexp.QuoteMeta(notFoundMessage+"\n"),
 947         )
 948 }
 949
 950 // Bad token in a cookie (even if it got there via our own
 951 // query-string-to-cookie redirect) is, in principle, retryable via
 952 // wb2-login-and-redirect flow.
 953 func (s *IntegrationSuite) TestVhostRedirectQueryTokenToBogusCookie(c *check.C) {
 954         // Inline
 955         resp := s.testVhostRedirectTokenToCookie(c, "GET",
 956                 arvadostest.FooCollection+".example.com/foo",
 957                 "?api_token=thisisabogustoken",
 958                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 959                 "",
 960                 http.StatusSeeOther,
 961                 "",
 962         )
 963         u, err := url.Parse(resp.Header().Get("Location"))
 964         c.Assert(err, check.IsNil)
 965         c.Logf("redirected to %s", u)
 966         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 967         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 968         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
 969
 970         // Download/attachment indicated by ?disposition=attachment
 971         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 972                 arvadostest.FooCollection+".example.com/foo",
 973                 "?api_token=thisisabogustoken&disposition=attachment",
 974                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 975                 "",
 976                 http.StatusSeeOther,
 977                 "",
 978         )
 979         u, err = url.Parse(resp.Header().Get("Location"))
 980         c.Assert(err, check.IsNil)
 981         c.Logf("redirected to %s", u)
 982         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 983         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
 984         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
 985
 986         // Download/attachment indicated by vhost
 987         resp = s.testVhostRedirectTokenToCookie(c, "GET",
 988                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
 989                 "?api_token=thisisabogustoken",
 990                 http.Header{"Sec-Fetch-Mode": {"navigate"}},
 991                 "",
 992                 http.StatusSeeOther,
 993                 "",
 994         )
 995         u, err = url.Parse(resp.Header().Get("Location"))
 996         c.Assert(err, check.IsNil)
 997         c.Logf("redirected to %s", u)
 998         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
 999         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "")
1000         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1001
1002         // Without "Sec-Fetch-Mode: navigate" header, just 401.
1003         s.testVhostRedirectTokenToCookie(c, "GET",
1004                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1005                 "?api_token=thisisabogustoken",
1006                 http.Header{"Sec-Fetch-Mode": {"cors"}},
1007                 "",
1008                 http.StatusUnauthorized,
1009                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
1010         )
1011         s.testVhostRedirectTokenToCookie(c, "GET",
1012                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host+"/c="+arvadostest.FooCollection+"/foo",
1013                 "?api_token=thisisabogustoken",
1014                 nil,
1015                 "",
1016                 http.StatusUnauthorized,
1017                 regexp.QuoteMeta(unauthorizedMessage+"\n"),
1018         )
1019 }
1020
1021 func (s *IntegrationSuite) TestVhostRedirectWithNoCache(c *check.C) {
1022         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1023                 arvadostest.FooCollection+".example.com/foo",
1024                 "?api_token=thisisabogustoken",
1025                 http.Header{
1026                         "Sec-Fetch-Mode": {"navigate"},
1027                         "Cache-Control":  {"no-cache"},
1028                 },
1029                 "",
1030                 http.StatusSeeOther,
1031                 "",
1032         )
1033         u, err := url.Parse(resp.Header().Get("Location"))
1034         c.Assert(err, check.IsNil)
1035         c.Logf("redirected to %s", u)
1036         c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1037         c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1038         c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
1039 }
1040
1041 func (s *IntegrationSuite) TestNoTokenWorkbench2LoginFlow(c *check.C) {
1042         for _, trial := range []struct {
1043                 anonToken    bool
1044                 cacheControl string
1045         }{
1046                 {},
1047                 {cacheControl: "no-cache"},
1048                 {anonToken: true},
1049                 {anonToken: true, cacheControl: "no-cache"},
1050         } {
1051                 c.Logf("trial: %+v", trial)
1052
1053                 if trial.anonToken {
1054                         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1055                 } else {
1056                         s.handler.Cluster.Users.AnonymousUserToken = ""
1057                 }
1058                 req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1059                 c.Assert(err, check.IsNil)
1060                 req.Header.Set("Sec-Fetch-Mode", "navigate")
1061                 if trial.cacheControl != "" {
1062                         req.Header.Set("Cache-Control", trial.cacheControl)
1063                 }
1064                 resp := httptest.NewRecorder()
1065                 s.handler.ServeHTTP(resp, req)
1066                 c.Check(resp.Code, check.Equals, http.StatusSeeOther)
1067                 u, err := url.Parse(resp.Header().Get("Location"))
1068                 c.Assert(err, check.IsNil)
1069                 c.Logf("redirected to %q", u)
1070                 c.Check(u.Host, check.Equals, s.handler.Cluster.Services.Workbench2.ExternalURL.Host)
1071                 c.Check(u.Query().Get("redirectToPreview"), check.Equals, "/c="+arvadostest.FooCollection+"/foo")
1072                 c.Check(u.Query().Get("redirectToDownload"), check.Equals, "")
1073         }
1074 }
1075
1076 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSingleOriginError(c *check.C) {
1077         s.testVhostRedirectTokenToCookie(c, "GET",
1078                 "example.com/c="+arvadostest.FooCollection+"/foo",
1079                 "?api_token="+arvadostest.ActiveToken,
1080                 nil,
1081                 "",
1082                 http.StatusBadRequest,
1083                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
1084         )
1085 }
1086
1087 // If client requests an attachment by putting ?disposition=attachment
1088 // in the query string, and gets redirected, the redirect target
1089 // should respond with an attachment.
1090 func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check.C) {
1091         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1092                 arvadostest.FooCollection+".example.com/foo",
1093                 "?disposition=attachment&api_token="+arvadostest.ActiveToken,
1094                 nil,
1095                 "",
1096                 http.StatusOK,
1097                 "foo",
1098         )
1099         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1100 }
1101
1102 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
1103         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1104         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1105                 "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
1106                 "?api_token="+arvadostest.ActiveToken,
1107                 nil,
1108                 "",
1109                 http.StatusOK,
1110                 "foo",
1111         )
1112         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1113 }
1114
1115 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
1116         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1117         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1118                 "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
1119                 "?api_token="+arvadostest.ActiveToken,
1120                 nil,
1121                 "",
1122                 http.StatusOK,
1123                 "waz",
1124         )
1125         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1126         resp = s.testVhostRedirectTokenToCookie(c, "GET",
1127                 "download.example.com/by_id/"+arvadostest.WazVersion1Collection+"/waz",
1128                 "?api_token="+arvadostest.ActiveToken,
1129                 nil,
1130                 "",
1131                 http.StatusOK,
1132                 "waz",
1133         )
1134         c.Check(resp.Header().Get("Content-Disposition"), check.Matches, "attachment(;.*)?")
1135 }
1136
1137 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
1138         s.handler.Cluster.Collections.TrustAllContent = true
1139         s.testVhostRedirectTokenToCookie(c, "GET",
1140                 "example.com/c="+arvadostest.FooCollection+"/foo",
1141                 "?api_token="+arvadostest.ActiveToken,
1142                 nil,
1143                 "",
1144                 http.StatusOK,
1145                 "foo",
1146         )
1147 }
1148
1149 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
1150         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
1151
1152         s.testVhostRedirectTokenToCookie(c, "GET",
1153                 "example.com/c="+arvadostest.FooCollection+"/foo",
1154                 "?api_token="+arvadostest.ActiveToken,
1155                 nil,
1156                 "",
1157                 http.StatusBadRequest,
1158                 regexp.QuoteMeta("cannot serve inline content at this URL (possible configuration error; see https://doc.arvados.org/install/install-keep-web.html#dns)\n"),
1159         )
1160
1161         resp := s.testVhostRedirectTokenToCookie(c, "GET",
1162                 "example.com:1234/c="+arvadostest.FooCollection+"/foo",
1163                 "?api_token="+arvadostest.ActiveToken,
1164                 nil,
1165                 "",
1166                 http.StatusOK,
1167                 "foo",
1168         )
1169         c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
1170 }
1171
1172 func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
1173         baseUrl := arvadostest.FooCollection + ".example.com/foo"
1174         query := url.Values{}
1175
1176         // The intent of these tests is to check that requests are redirected
1177         // correctly in the presence of multiple API tokens. The exact response
1178         // codes and content are not closely considered: they're just how
1179         // keep-web responded when we made the smallest possible fix. Changing
1180         // those responses may be okay, but you should still test all these
1181         // different cases and the associated redirect logic.
1182         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
1183         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1184         query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
1185         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1186         query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
1187         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1188         query["api_token"] = []string{"", arvadostest.ActiveToken}
1189         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
1190
1191         expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
1192         query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
1193         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1194         query["api_token"] = []string{arvadostest.AnonymousToken, ""}
1195         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1196         query["api_token"] = []string{"", arvadostest.AnonymousToken}
1197         s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
1198 }
1199
1200 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
1201         s.testVhostRedirectTokenToCookie(c, "POST",
1202                 arvadostest.FooCollection+".example.com/foo",
1203                 "",
1204                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
1205                 url.Values{"api_token": {arvadostest.ActiveToken}}.Encode(),
1206                 http.StatusOK,
1207                 "foo",
1208         )
1209 }
1210
1211 func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C) {
1212         s.testVhostRedirectTokenToCookie(c, "POST",
1213                 arvadostest.FooCollection+".example.com/foo",
1214                 "",
1215                 http.Header{"Content-Type": {"application/x-www-form-urlencoded"}},
1216                 url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
1217                 http.StatusNotFound,
1218                 regexp.QuoteMeta(notFoundMessage+"\n"),
1219         )
1220 }
1221
1222 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
1223         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1224         s.testVhostRedirectTokenToCookie(c, "GET",
1225                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
1226                 "",
1227                 nil,
1228                 "",
1229                 http.StatusOK,
1230                 "Hello world\n",
1231         )
1232 }
1233
1234 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
1235         s.handler.Cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
1236         s.testVhostRedirectTokenToCookie(c, "GET",
1237                 "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
1238                 "",
1239                 nil,
1240                 "",
1241                 http.StatusUnauthorized,
1242                 "Authorization tokens are not accepted here: .*\n",
1243         )
1244 }
1245
1246 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
1247         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1248
1249         client := arvados.NewClientFromEnv()
1250         client.AuthToken = arvadostest.ActiveToken
1251         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
1252         c.Assert(err, check.IsNil)
1253         path := `https:\\"odd' path chars`
1254         f, err := fs.OpenFile(path, os.O_CREATE, 0777)
1255         c.Assert(err, check.IsNil)
1256         f.Close()
1257         mtxt, err := fs.MarshalManifest(".")
1258         c.Assert(err, check.IsNil)
1259         var coll arvados.Collection
1260         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1261                 "collection": map[string]string{
1262                         "manifest_text": mtxt,
1263                 },
1264         })
1265         c.Assert(err, check.IsNil)
1266
1267         u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
1268         req := &http.Request{
1269                 Method:     "GET",
1270                 Host:       u.Host,
1271                 URL:        u,
1272                 RequestURI: u.RequestURI(),
1273                 Header: http.Header{
1274                         "Authorization": {"Bearer " + client.AuthToken},
1275                 },
1276         }
1277         resp := httptest.NewRecorder()
1278         s.handler.ServeHTTP(resp, req)
1279         c.Check(resp.Code, check.Equals, http.StatusOK)
1280         doc, err := html.Parse(resp.Body)
1281         c.Assert(err, check.IsNil)
1282         pathHrefMap := getPathHrefMap(doc)
1283         c.Check(pathHrefMap, check.HasLen, 1) // the one leaf added to collection
1284         href, hasPath := pathHrefMap[path]
1285         c.Assert(hasPath, check.Equals, true) // the path is listed
1286         relUrl := mustParseURL(href)
1287         c.Check(relUrl.Path, check.Equals, "./"+path) // href can be decoded back to path
1288 }
1289
1290 func (s *IntegrationSuite) TestForwardSlashSubstitution(c *check.C) {
1291         arv := arvados.NewClientFromEnv()
1292         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1293         s.handler.Cluster.Collections.ForwardSlashNameSubstitution = "{SOLIDUS}"
1294         name := "foo/bar/baz"
1295         nameShown := strings.Replace(name, "/", "{SOLIDUS}", -1)
1296
1297         client := arvados.NewClientFromEnv()
1298         client.AuthToken = arvadostest.ActiveToken
1299         fs, err := (&arvados.Collection{}).FileSystem(client, nil)
1300         c.Assert(err, check.IsNil)
1301         f, err := fs.OpenFile("filename", os.O_CREATE, 0777)
1302         c.Assert(err, check.IsNil)
1303         f.Close()
1304         mtxt, err := fs.MarshalManifest(".")
1305         c.Assert(err, check.IsNil)
1306         var coll arvados.Collection
1307         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1308                 "collection": map[string]string{
1309                         "manifest_text": mtxt,
1310                         "name":          name,
1311                         "owner_uuid":    arvadostest.AProjectUUID,
1312                 },
1313         })
1314         c.Assert(err, check.IsNil)
1315         defer arv.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
1316
1317         base := "http://download.example.com/by_id/" + coll.OwnerUUID + "/"
1318         for tryURL, expectedAnchorText := range map[string]string{
1319                 base:                   nameShown + "/",
1320                 base + nameShown + "/": "filename",
1321         } {
1322                 u, _ := url.Parse(tryURL)
1323                 req := &http.Request{
1324                         Method:     "GET",
1325                         Host:       u.Host,
1326                         URL:        u,
1327                         RequestURI: u.RequestURI(),
1328                         Header: http.Header{
1329                                 "Authorization": {"Bearer " + client.AuthToken},
1330                         },
1331                 }
1332                 resp := httptest.NewRecorder()
1333                 s.handler.ServeHTTP(resp, req)
1334                 c.Check(resp.Code, check.Equals, http.StatusOK)
1335                 doc, err := html.Parse(resp.Body)
1336                 c.Assert(err, check.IsNil) // valid HTML
1337                 pathHrefMap := getPathHrefMap(doc)
1338                 href, hasExpected := pathHrefMap[expectedAnchorText]
1339                 c.Assert(hasExpected, check.Equals, true) // has expected anchor text
1340                 c.Assert(href, check.Not(check.Equals), "")
1341                 relUrl := mustParseURL(href)
1342                 c.Check(relUrl.Path, check.Equals, "./"+expectedAnchorText) // decoded href maps back to the anchor text
1343         }
1344 }
1345
1346 // XHRs can't follow redirect-with-cookie so they rely on method=POST
1347 // and disposition=attachment (telling us it's acceptable to respond
1348 // with content instead of a redirect) and an Origin header that gets
1349 // added automatically by the browser (telling us it's desirable to do
1350 // so).
1351 func (s *IntegrationSuite) TestXHRNoRedirect(c *check.C) {
1352         u, _ := url.Parse("http://example.com/c=" + arvadostest.FooCollection + "/foo")
1353         req := &http.Request{
1354                 Method:     "POST",
1355                 Host:       u.Host,
1356                 URL:        u,
1357                 RequestURI: u.RequestURI(),
1358                 Header: http.Header{
1359                         "Origin":       {"https://origin.example"},
1360                         "Content-Type": {"application/x-www-form-urlencoded"},
1361                 },
1362                 Body: ioutil.NopCloser(strings.NewReader(url.Values{
1363                         "api_token":   {arvadostest.ActiveToken},
1364                         "disposition": {"attachment"},
1365                 }.Encode())),
1366         }
1367         resp := httptest.NewRecorder()
1368         s.handler.ServeHTTP(resp, req)
1369         c.Check(resp.Code, check.Equals, http.StatusOK)
1370         c.Check(resp.Body.String(), check.Equals, "foo")
1371         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1372
1373         // GET + Origin header is representative of both AJAX GET
1374         // requests and inline images via <IMG crossorigin="anonymous"
1375         // src="...">.
1376         u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
1377         req = &http.Request{
1378                 Method:     "GET",
1379                 Host:       u.Host,
1380                 URL:        u,
1381                 RequestURI: u.RequestURI(),
1382                 Header: http.Header{
1383                         "Origin": {"https://origin.example"},
1384                 },
1385         }
1386         resp = httptest.NewRecorder()
1387         s.handler.ServeHTTP(resp, req)
1388         c.Check(resp.Code, check.Equals, http.StatusOK)
1389         c.Check(resp.Body.String(), check.Equals, "foo")
1390         c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
1391 }
1392
1393 func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString string, reqHeader http.Header, reqBody string, expectStatus int, matchRespBody string) *httptest.ResponseRecorder {
1394         if reqHeader == nil {
1395                 reqHeader = http.Header{}
1396         }
1397         u, _ := url.Parse(`http://` + hostPath + queryString)
1398         c.Logf("requesting %s", u)
1399         req := &http.Request{
1400                 Method:     method,
1401                 Host:       u.Host,
1402                 URL:        u,
1403                 RequestURI: u.RequestURI(),
1404                 Header:     reqHeader,
1405                 Body:       ioutil.NopCloser(strings.NewReader(reqBody)),
1406         }
1407
1408         resp := httptest.NewRecorder()
1409         defer func() {
1410                 c.Check(resp.Code, check.Equals, expectStatus)
1411                 c.Check(resp.Body.String(), check.Matches, matchRespBody)
1412         }()
1413
1414         s.handler.ServeHTTP(resp, req)
1415         if resp.Code != http.StatusSeeOther {
1416                 attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
1417                 // Since we're not redirecting, check that any api_token in the URL is
1418                 // handled safely.
1419                 // If there is no token in the URL, then we're good.
1420                 // Otherwise, if the response code is an error, the body is expected to
1421                 // be static content, and nothing that might maliciously introspect the
1422                 // URL. It's considered safe and allowed.
1423                 // Otherwise, if the response content has attachment disposition,
1424                 // that's considered safe for all the reasons explained in the
1425                 // safeAttachment comment in handler.go.
1426                 c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
1427                 return resp
1428         }
1429
1430         loc, err := url.Parse(resp.Header().Get("Location"))
1431         c.Assert(err, check.IsNil)
1432         c.Check(loc.Scheme, check.Equals, u.Scheme)
1433         c.Check(loc.Host, check.Equals, u.Host)
1434         c.Check(loc.RawPath, check.Equals, u.RawPath)
1435         // If the response was a redirect, it should never include an API token.
1436         c.Check(loc.Query().Has("api_token"), check.Equals, false)
1437         c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
1438         cookies := (&http.Response{Header: resp.Header()}).Cookies()
1439
1440         c.Logf("following redirect to %s", u)
1441         req = &http.Request{
1442                 Method:     "GET",
1443                 Host:       loc.Host,
1444                 URL:        loc,
1445                 RequestURI: loc.RequestURI(),
1446                 Header:     reqHeader,
1447         }
1448         for _, c := range cookies {
1449                 req.AddCookie(c)
1450         }
1451
1452         resp = httptest.NewRecorder()
1453         s.handler.ServeHTTP(resp, req)
1454
1455         if resp.Code != http.StatusSeeOther {
1456                 c.Check(resp.Header().Get("Location"), check.Equals, "")
1457         }
1458         return resp
1459 }
1460
1461 func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
1462         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
1463         s.testDirectoryListing(c)
1464 }
1465
1466 func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
1467         s.handler.Cluster.Users.AnonymousUserToken = ""
1468         s.testDirectoryListing(c)
1469 }
1470
1471 func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
1472         // The "ownership cycle" test fixtures are reachable from the
1473         // "filter group without filters" group, causing webdav's
1474         // walkfs to recurse indefinitely. Avoid that by deleting one
1475         // of the bogus fixtures.
1476         arv := arvados.NewClientFromEnv()
1477         err := arv.RequestAndDecode(nil, "DELETE", "arvados/v1/groups/zzzzz-j7d0g-cx2al9cqkmsf1hs", nil, nil)
1478         if err != nil {
1479                 c.Assert(err, check.FitsTypeOf, &arvados.TransactionError{})
1480                 c.Check(err.(*arvados.TransactionError).StatusCode, check.Equals, 404)
1481         }
1482
1483         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1484         authHeader := http.Header{
1485                 "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
1486         }
1487         for _, trial := range []struct {
1488                 uri      string
1489                 header   http.Header
1490                 expect   []string
1491                 redirect string
1492                 cutDirs  int
1493         }{
1494                 {
1495                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/",
1496                         header:  authHeader,
1497                         expect:  []string{"dir1/foo", "dir1/bar"},
1498                         cutDirs: 0,
1499                 },
1500                 {
1501                         uri:     strings.Replace(arvadostest.FooAndBarFilesInDirPDH, "+", "-", -1) + ".example.com/dir1/",
1502                         header:  authHeader,
1503                         expect:  []string{"foo", "bar"},
1504                         cutDirs: 1,
1505                 },
1506                 {
1507                         // URLs of this form ignore authHeader, and
1508                         // FooAndBarFilesInDirUUID isn't public, so
1509                         // this returns 401.
1510                         uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
1511                         header: authHeader,
1512                         expect: nil,
1513                 },
1514                 {
1515                         uri:     "download.example.com/users/active/foo_file_in_dir/",
1516                         header:  authHeader,
1517                         expect:  []string{"dir1/"},
1518                         cutDirs: 3,
1519                 },
1520                 {
1521                         uri:     "download.example.com/users/active/foo_file_in_dir/dir1/",
1522                         header:  authHeader,
1523                         expect:  []string{"bar"},
1524                         cutDirs: 4,
1525                 },
1526                 {
1527                         uri:     "download.example.com/",
1528                         header:  authHeader,
1529                         expect:  []string{"users/"},
1530                         cutDirs: 0,
1531                 },
1532                 {
1533                         uri:      "download.example.com/users",
1534                         header:   authHeader,
1535                         redirect: "/users/",
1536                         expect:   []string{"active/"},
1537                         cutDirs:  1,
1538                 },
1539                 {
1540                         uri:     "download.example.com/users/",
1541                         header:  authHeader,
1542                         expect:  []string{"active/"},
1543                         cutDirs: 1,
1544                 },
1545                 {
1546                         uri:      "download.example.com/users/active",
1547                         header:   authHeader,
1548                         redirect: "/users/active/",
1549                         expect:   []string{"foo_file_in_dir/"},
1550                         cutDirs:  2,
1551                 },
1552                 {
1553                         uri:     "download.example.com/users/active/",
1554                         header:  authHeader,
1555                         expect:  []string{"foo_file_in_dir/"},
1556                         cutDirs: 2,
1557                 },
1558                 {
1559                         uri:     "collections.example.com/collections/download/" + arvadostest.FooAndBarFilesInDirUUID + "/" + arvadostest.ActiveToken + "/",
1560                         header:  nil,
1561                         expect:  []string{"dir1/foo", "dir1/bar"},
1562                         cutDirs: 4,
1563                 },
1564                 {
1565                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken + "/",
1566                         header:  nil,
1567                         expect:  []string{"dir1/foo", "dir1/bar"},
1568                         cutDirs: 2,
1569                 },
1570                 {
1571                         uri:     "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/t=" + arvadostest.ActiveToken,
1572                         header:  nil,
1573                         expect:  []string{"dir1/foo", "dir1/bar"},
1574                         cutDirs: 2,
1575                 },
1576                 {
1577                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID,
1578                         header:  authHeader,
1579                         expect:  []string{"dir1/foo", "dir1/bar"},
1580                         cutDirs: 1,
1581                 },
1582                 {
1583                         uri:      "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1",
1584                         header:   authHeader,
1585                         redirect: "/c=" + arvadostest.FooAndBarFilesInDirUUID + "/dir1/",
1586                         expect:   []string{"foo", "bar"},
1587                         cutDirs:  2,
1588                 },
1589                 {
1590                         uri:     "download.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/_/dir1/",
1591                         header:  authHeader,
1592                         expect:  []string{"foo", "bar"},
1593                         cutDirs: 3,
1594                 },
1595                 {
1596                         uri:      arvadostest.FooAndBarFilesInDirUUID + ".example.com/dir1?api_token=" + arvadostest.ActiveToken,
1597                         header:   authHeader,
1598                         redirect: "/dir1/",
1599                         expect:   []string{"foo", "bar"},
1600                         cutDirs:  1,
1601                 },
1602                 {
1603                         uri:    "collections.example.com/c=" + arvadostest.FooAndBarFilesInDirUUID + "/theperthcountyconspiracydoesnotexist/",
1604                         header: authHeader,
1605                         expect: nil,
1606                 },
1607                 {
1608                         uri:     "download.example.com/c=" + arvadostest.WazVersion1Collection,
1609                         header:  authHeader,
1610                         expect:  []string{"waz"},
1611                         cutDirs: 1,
1612                 },
1613                 {
1614                         uri:     "download.example.com/by_id/" + arvadostest.WazVersion1Collection,
1615                         header:  authHeader,
1616                         expect:  []string{"waz"},
1617                         cutDirs: 2,
1618                 },
1619                 {
1620                         uri:     "download.example.com/users/active/This filter group/",
1621                         header:  authHeader,
1622                         expect:  []string{"A Subproject/"},
1623                         cutDirs: 3,
1624                 },
1625                 {
1626                         uri:     "download.example.com/users/active/This filter group/A Subproject",
1627                         header:  authHeader,
1628                         expect:  []string{"baz_file/"},
1629                         cutDirs: 4,
1630                 },
1631                 {
1632                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID,
1633                         header:  authHeader,
1634                         expect:  []string{"A Subproject/"},
1635                         cutDirs: 2,
1636                 },
1637                 {
1638                         uri:     "download.example.com/by_id/" + arvadostest.AFilterGroupUUID + "/A Subproject",
1639                         header:  authHeader,
1640                         expect:  []string{"baz_file/"},
1641                         cutDirs: 3,
1642                 },
1643         } {
1644                 comment := check.Commentf("HTML: %q redir %q => %q", trial.uri, trial.redirect, trial.expect)
1645                 resp := httptest.NewRecorder()
1646                 u := mustParseURL("//" + trial.uri)
1647                 req := &http.Request{
1648                         Method:     "GET",
1649                         Host:       u.Host,
1650                         URL:        u,
1651                         RequestURI: u.RequestURI(),
1652                         Header:     copyHeader(trial.header),
1653                 }
1654                 s.handler.ServeHTTP(resp, req)
1655                 var cookies []*http.Cookie
1656                 for resp.Code == http.StatusSeeOther {
1657                         u, _ := req.URL.Parse(resp.Header().Get("Location"))
1658                         req = &http.Request{
1659                                 Method:     "GET",
1660                                 Host:       u.Host,
1661                                 URL:        u,
1662                                 RequestURI: u.RequestURI(),
1663                                 Header:     copyHeader(trial.header),
1664                         }
1665                         cookies = append(cookies, (&http.Response{Header: resp.Header()}).Cookies()...)
1666                         for _, c := range cookies {
1667                                 req.AddCookie(c)
1668                         }
1669                         resp = httptest.NewRecorder()
1670                         s.handler.ServeHTTP(resp, req)
1671                 }
1672                 if trial.redirect != "" {
1673                         c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
1674                 }
1675                 if trial.expect == nil {
1676                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1677                 } else {
1678                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1679                         listingPageDoc, err := html.Parse(resp.Body)
1680                         c.Check(err, check.IsNil, comment) // valid HTML document
1681                         pathHrefMap := getPathHrefMap(listingPageDoc)
1682                         c.Assert(pathHrefMap, check.Not(check.HasLen), 0, comment)
1683                         for _, e := range trial.expect {
1684                                 href, hasE := pathHrefMap[e]
1685                                 c.Check(hasE, check.Equals, true, comment) // expected path is listed
1686                                 relUrl := mustParseURL(href)
1687                                 c.Check(relUrl.Path, check.Equals, "./"+e, comment) // href can be decoded back to path
1688                         }
1689                         wgetCommand := getWgetExamplePre(listingPageDoc)
1690                         wgetExpected := regexp.MustCompile(`^\$ wget .*--cut-dirs=(\d+) .*'(https?://[^']+)'$`)
1691                         wgetMatchGroups := wgetExpected.FindStringSubmatch(wgetCommand)
1692                         c.Assert(wgetMatchGroups, check.NotNil)                                     // wget command matches
1693                         c.Check(wgetMatchGroups[1], check.Equals, fmt.Sprintf("%d", trial.cutDirs)) // correct level of cut dirs in wget command
1694                         printedUrl := mustParseURL(wgetMatchGroups[2])
1695                         c.Check(printedUrl.Host, check.Equals, req.URL.Host)
1696                         c.Check(printedUrl.Path, check.Equals, req.URL.Path) // URL arg in wget command can be decoded to the right path
1697                 }
1698
1699                 comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
1700                 req = &http.Request{
1701                         Method:     "OPTIONS",
1702                         Host:       u.Host,
1703                         URL:        u,
1704                         RequestURI: u.RequestURI(),
1705                         Header:     copyHeader(trial.header),
1706                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1707                 }
1708                 resp = httptest.NewRecorder()
1709                 s.handler.ServeHTTP(resp, req)
1710                 if trial.expect == nil {
1711                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1712                 } else {
1713                         c.Check(resp.Code, check.Equals, http.StatusOK, comment)
1714                 }
1715
1716                 req = &http.Request{
1717                         Method:     "PROPFIND",
1718                         Host:       u.Host,
1719                         URL:        u,
1720                         RequestURI: u.RequestURI(),
1721                         Header:     copyHeader(trial.header),
1722                         Body:       ioutil.NopCloser(&bytes.Buffer{}),
1723                 }
1724                 resp = httptest.NewRecorder()
1725                 s.handler.ServeHTTP(resp, req)
1726                 // This check avoids logging a big XML document in the
1727                 // event webdav throws a 500 error after sending
1728                 // headers for a 207.
1729                 if !c.Check(strings.HasSuffix(resp.Body.String(), "Internal Server Error"), check.Equals, false) {
1730                         continue
1731                 }
1732                 if trial.expect == nil {
1733                         c.Check(resp.Code, check.Equals, http.StatusUnauthorized, comment)
1734                 } else {
1735                         c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
1736                         for _, e := range trial.expect {
1737                                 if strings.HasSuffix(e, "/") {
1738                                         e = filepath.Join(u.Path, e) + "/"
1739                                 } else {
1740                                         e = filepath.Join(u.Path, e)
1741                                 }
1742                                 e = strings.Replace(e, " ", "%20", -1)
1743                                 c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+e+`</D:href>.*`, comment)
1744                         }
1745                 }
1746         }
1747 }
1748
1749 // Shallow-traverse the HTML document, gathering the nodes satisfying the
1750 // predicate function in the output slice. If a node matches the predicate,
1751 // none of its children will be visited.
1752 func getNodes(document *html.Node, predicate func(*html.Node) bool) []*html.Node {
1753         var acc []*html.Node
1754         var traverse func(*html.Node, []*html.Node) []*html.Node
1755         traverse = func(root *html.Node, sofar []*html.Node) []*html.Node {
1756                 if root == nil {
1757                         return sofar
1758                 }
1759                 if predicate(root) {
1760                         return append(sofar, root)
1761                 }
1762                 for cur := root.FirstChild; cur != nil; cur = cur.NextSibling {
1763                         sofar = traverse(cur, sofar)
1764                 }
1765                 return sofar
1766         }
1767         return traverse(document, acc)
1768 }
1769
1770 // Returns true if a node has the attribute targetAttr with the given value
1771 func matchesAttributeValue(node *html.Node, targetAttr string, value string) bool {
1772         for _, attr := range node.Attr {
1773                 if attr.Key == targetAttr && attr.Val == value {
1774                         return true
1775                 }
1776         }
1777         return false
1778 }
1779
1780 // Concatenate the content of text-node children of node; only direct
1781 // children are visited, and any non-text children are skipped.
1782 func getNodeText(node *html.Node) string {
1783         var recv strings.Builder
1784         for c := node.FirstChild; c != nil; c = c.NextSibling {
1785                 if c.Type == html.TextNode {
1786                         recv.WriteString(c.Data)
1787                 }
1788         }
1789         return recv.String()
1790 }
1791
1792 // Returns a map from the directory listing item string (a path) to the href
1793 // value of its <a> tag (an encoded relative URL)
1794 func getPathHrefMap(document *html.Node) map[string]string {
1795         isItemATag := func(node *html.Node) bool {
1796                 return node.Type == html.ElementNode && node.Data == "a" && matchesAttributeValue(node, "class", "item")
1797         }
1798         aTags := getNodes(document, isItemATag)
1799         output := make(map[string]string)
1800         for _, elem := range aTags {
1801                 textContent := getNodeText(elem)
1802                 for _, attr := range elem.Attr {
1803                         if attr.Key == "href" {
1804                                 output[textContent] = attr.Val
1805                                 break
1806                         }
1807                 }
1808         }
1809         return output
1810 }
1811
1812 func getWgetExamplePre(document *html.Node) string {
1813         isWgetPre := func(node *html.Node) bool {
1814                 return node.Type == html.ElementNode && matchesAttributeValue(node, "id", "wget-example")
1815         }
1816         elements := getNodes(document, isWgetPre)
1817         if len(elements) != 1 {
1818                 return ""
1819         }
1820         return getNodeText(elements[0])
1821 }
1822
1823 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
1824         arv := arvados.NewClientFromEnv()
1825         var newCollection arvados.Collection
1826         err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1827                 "collection": map[string]string{
1828                         "owner_uuid":    arvadostest.ActiveUserUUID,
1829                         "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
1830                         "name":          "keep-web test collection",
1831                 },
1832                 "ensure_unique_name": true,
1833         })
1834         c.Assert(err, check.IsNil)
1835         defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1836
1837         var updated arvados.Collection
1838         for _, fnm := range []string{"foo.txt", "bar.txt"} {
1839                 s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
1840                 u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
1841                 req := &http.Request{
1842                         Method:     "DELETE",
1843                         Host:       u.Host,
1844                         URL:        u,
1845                         RequestURI: u.RequestURI(),
1846                         Header: http.Header{
1847                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
1848                         },
1849                 }
1850                 resp := httptest.NewRecorder()
1851                 s.handler.ServeHTTP(resp, req)
1852                 c.Check(resp.Code, check.Equals, http.StatusNoContent)
1853
1854                 updated = arvados.Collection{}
1855                 err = arv.RequestAndDecode(&updated, "GET", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
1856                 c.Check(err, check.IsNil)
1857                 c.Check(updated.ManifestText, check.Not(check.Matches), `(?ms).*\Q`+fnm+`\E.*`)
1858                 c.Logf("updated manifest_text %q", updated.ManifestText)
1859         }
1860         c.Check(updated.ManifestText, check.Equals, "")
1861 }
1862
1863 func (s *IntegrationSuite) TestFileContentType(c *check.C) {
1864         s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
1865
1866         client := arvados.NewClientFromEnv()
1867         client.AuthToken = arvadostest.ActiveToken
1868         arv, err := arvadosclient.New(client)
1869         c.Assert(err, check.Equals, nil)
1870         kc, err := keepclient.MakeKeepClient(arv)
1871         c.Assert(err, check.Equals, nil)
1872
1873         fs, err := (&arvados.Collection{}).FileSystem(client, kc)
1874         c.Assert(err, check.IsNil)
1875
1876         trials := []struct {
1877                 filename    string
1878                 content     string
1879                 contentType string
1880         }{
1881                 {"picture.txt", "BMX bikes are small this year\n", "text/plain; charset=utf-8"},
1882                 {"picture.bmp", "BMX bikes are small this year\n", "image/(x-ms-)?bmp"},
1883                 {"picture.jpg", "BMX bikes are small this year\n", "image/jpeg"},
1884                 {"picture1", "BMX bikes are small this year\n", "image/bmp"},            // content sniff; "BM" is the magic signature for .bmp
1885                 {"picture2", "Cars are small this year\n", "text/plain; charset=utf-8"}, // content sniff
1886         }
1887         for _, trial := range trials {
1888                 f, err := fs.OpenFile(trial.filename, os.O_CREATE|os.O_WRONLY, 0777)
1889                 c.Assert(err, check.IsNil)
1890                 _, err = f.Write([]byte(trial.content))
1891                 c.Assert(err, check.IsNil)
1892                 c.Assert(f.Close(), check.IsNil)
1893         }
1894         mtxt, err := fs.MarshalManifest(".")
1895         c.Assert(err, check.IsNil)
1896         var coll arvados.Collection
1897         err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
1898                 "collection": map[string]string{
1899                         "manifest_text": mtxt,
1900                 },
1901         })
1902         c.Assert(err, check.IsNil)
1903
1904         for _, trial := range trials {
1905                 u, _ := url.Parse("http://download.example.com/by_id/" + coll.UUID + "/" + trial.filename)
1906                 req := &http.Request{
1907                         Method:     "GET",
1908                         Host:       u.Host,
1909                         URL:        u,
1910                         RequestURI: u.RequestURI(),
1911                         Header: http.Header{
1912                                 "Authorization": {"Bearer " + client.AuthToken},
1913                         },
1914                 }
1915                 resp := httptest.NewRecorder()
1916                 s.handler.ServeHTTP(resp, req)
1917                 c.Check(resp.Code, check.Equals, http.StatusOK)
1918                 c.Check(resp.Header().Get("Content-Type"), check.Matches, trial.contentType)
1919                 c.Check(resp.Body.String(), check.Equals, trial.content)
1920         }
1921 }
1922
1923 func (s *IntegrationSuite) TestCacheSize(c *check.C) {
1924         req, err := http.NewRequest("GET", "http://"+arvadostest.FooCollection+".example.com/foo", nil)
1925         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
1926         c.Assert(err, check.IsNil)
1927         resp := httptest.NewRecorder()
1928         s.handler.ServeHTTP(resp, req)
1929         c.Assert(resp.Code, check.Equals, http.StatusOK)
1930         c.Check(s.handler.Cache.sessions[arvadostest.ActiveTokenV2].client.DiskCacheSize.Percent(), check.Equals, int64(10))
1931 }
1932
1933 // Writing to a collection shouldn't affect its entry in the
1934 // PDH-to-manifest cache.
1935 func (s *IntegrationSuite) TestCacheWriteCollectionSamePDH(c *check.C) {
1936         arv, err := arvadosclient.MakeArvadosClient()
1937         c.Assert(err, check.Equals, nil)
1938         arv.ApiToken = arvadostest.ActiveToken
1939
1940         u := mustParseURL("http://x.example/testfile")
1941         req := &http.Request{
1942                 Method:     "GET",
1943                 Host:       u.Host,
1944                 URL:        u,
1945                 RequestURI: u.RequestURI(),
1946                 Header:     http.Header{"Authorization": {"Bearer " + arv.ApiToken}},
1947         }
1948
1949         checkWithID := func(id string, status int) {
1950                 req.URL.Host = strings.Replace(id, "+", "-", -1) + ".example"
1951                 req.Host = req.URL.Host
1952                 resp := httptest.NewRecorder()
1953                 s.handler.ServeHTTP(resp, req)
1954                 c.Check(resp.Code, check.Equals, status)
1955         }
1956
1957         var colls [2]arvados.Collection
1958         for i := range colls {
1959                 err := arv.Create("collections",
1960                         map[string]interface{}{
1961                                 "ensure_unique_name": true,
1962                                 "collection": map[string]interface{}{
1963                                         "name": "test collection",
1964                                 },
1965                         }, &colls[i])
1966                 c.Assert(err, check.Equals, nil)
1967         }
1968
1969         // Populate cache with empty collection
1970         checkWithID(colls[0].PortableDataHash, http.StatusNotFound)
1971
1972         // write a file to colls[0]
1973         reqPut := *req
1974         reqPut.Method = "PUT"
1975         reqPut.URL.Host = colls[0].UUID + ".example"
1976         reqPut.Host = req.URL.Host
1977         reqPut.Body = ioutil.NopCloser(bytes.NewBufferString("testdata"))
1978         resp := httptest.NewRecorder()
1979         s.handler.ServeHTTP(resp, &reqPut)
1980         c.Check(resp.Code, check.Equals, http.StatusCreated)
1981
1982         // new file should not appear in colls[1]
1983         checkWithID(colls[1].PortableDataHash, http.StatusNotFound)
1984         checkWithID(colls[1].UUID, http.StatusNotFound)
1985
1986         checkWithID(colls[0].UUID, http.StatusOK)
1987 }
1988
1989 func copyHeader(h http.Header) http.Header {
1990         hc := http.Header{}
1991         for k, v := range h {
1992                 hc[k] = append([]string(nil), v...)
1993         }
1994         return hc
1995 }
1996
1997 func (s *IntegrationSuite) checkUploadDownloadRequest(c *check.C, req *http.Request,
1998         successCode int, direction string, perm bool, userUuid, collectionUuid, collectionPDH, filepath string) {
1999
2000         client := arvados.NewClientFromEnv()
2001         client.AuthToken = arvadostest.AdminToken
2002         var logentries arvados.LogList
2003         limit1 := 1
2004         err := client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
2005                 arvados.ResourceListParams{
2006                         Limit: &limit1,
2007                         Order: "created_at desc"})
2008         c.Check(err, check.IsNil)
2009         c.Check(logentries.Items, check.HasLen, 1)
2010         lastLogId := logentries.Items[0].ID
2011         c.Logf("lastLogId: %d", lastLogId)
2012
2013         var logbuf bytes.Buffer
2014         logger := logrus.New()
2015         logger.Out = &logbuf
2016         resp := httptest.NewRecorder()
2017         req = req.WithContext(ctxlog.Context(context.Background(), logger))
2018         s.handler.ServeHTTP(resp, req)
2019
2020         if perm {
2021                 c.Check(resp.Result().StatusCode, check.Equals, successCode)
2022                 c.Check(logbuf.String(), check.Matches, `(?ms).*msg="File `+direction+`".*`)
2023                 c.Check(logbuf.String(), check.Not(check.Matches), `(?ms).*level=error.*`)
2024
2025                 deadline := time.Now().Add(time.Second)
2026                 for {
2027                         c.Assert(time.Now().After(deadline), check.Equals, false, check.Commentf("timed out waiting for log entry"))
2028                         logentries = arvados.LogList{}
2029                         err = client.RequestAndDecode(&logentries, "GET", "arvados/v1/logs", nil,
2030                                 arvados.ResourceListParams{
2031                                         Filters: []arvados.Filter{
2032                                                 {Attr: "event_type", Operator: "=", Operand: "file_" + direction},
2033                                                 {Attr: "object_uuid", Operator: "=", Operand: userUuid},
2034                                         },
2035                                         Limit: &limit1,
2036                                         Order: "created_at desc",
2037                                 })
2038                         c.Assert(err, check.IsNil)
2039                         if len(logentries.Items) > 0 &&
2040                                 logentries.Items[0].ID > lastLogId &&
2041                                 logentries.Items[0].ObjectUUID == userUuid &&
2042                                 logentries.Items[0].Properties["collection_uuid"] == collectionUuid &&
2043                                 (collectionPDH == "" || logentries.Items[0].Properties["portable_data_hash"] == collectionPDH) &&
2044                                 logentries.Items[0].Properties["collection_file_path"] == filepath {
2045                                 break
2046                         }
2047                         c.Logf("logentries.Items: %+v", logentries.Items)
2048                         time.Sleep(50 * time.Millisecond)
2049                 }
2050         } else {
2051                 c.Check(resp.Result().StatusCode, check.Equals, http.StatusForbidden)
2052                 c.Check(logbuf.String(), check.Equals, "")
2053         }
2054 }
2055
2056 func (s *IntegrationSuite) TestDownloadLoggingPermission(c *check.C) {
2057         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
2058
2059         s.handler.Cluster.Collections.TrustAllContent = true
2060
2061         for _, adminperm := range []bool{true, false} {
2062                 for _, userperm := range []bool{true, false} {
2063                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Download = adminperm
2064                         s.handler.Cluster.Collections.WebDAVPermission.User.Download = userperm
2065
2066                         // Test admin permission
2067                         req := &http.Request{
2068                                 Method:     "GET",
2069                                 Host:       u.Host,
2070                                 URL:        u,
2071                                 RequestURI: u.RequestURI(),
2072                                 Header: http.Header{
2073                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
2074                                 },
2075                         }
2076                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", adminperm,
2077                                 arvadostest.AdminUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
2078
2079                         // Test user permission
2080                         req = &http.Request{
2081                                 Method:     "GET",
2082                                 Host:       u.Host,
2083                                 URL:        u,
2084                                 RequestURI: u.RequestURI(),
2085                                 Header: http.Header{
2086                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2087                                 },
2088                         }
2089                         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", userperm,
2090                                 arvadostest.ActiveUserUUID, arvadostest.FooCollection, arvadostest.FooCollectionPDH, "foo")
2091                 }
2092         }
2093
2094         s.handler.Cluster.Collections.WebDAVPermission.User.Download = true
2095
2096         for _, tryurl := range []string{"http://" + arvadostest.MultilevelCollection1 + ".keep-web.example/dir1/subdir/file1",
2097                 "http://keep-web/users/active/multilevel_collection_1/dir1/subdir/file1"} {
2098
2099                 u = mustParseURL(tryurl)
2100                 req := &http.Request{
2101                         Method:     "GET",
2102                         Host:       u.Host,
2103                         URL:        u,
2104                         RequestURI: u.RequestURI(),
2105                         Header: http.Header{
2106                                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
2107                         },
2108                 }
2109                 s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
2110                         arvadostest.ActiveUserUUID, arvadostest.MultilevelCollection1, arvadostest.MultilevelCollection1PDH, "dir1/subdir/file1")
2111         }
2112
2113         u = mustParseURL("http://" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".keep-web.example/foo")
2114         req := &http.Request{
2115                 Method:     "GET",
2116                 Host:       u.Host,
2117                 URL:        u,
2118                 RequestURI: u.RequestURI(),
2119                 Header: http.Header{
2120                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2121                 },
2122         }
2123         s.checkUploadDownloadRequest(c, req, http.StatusOK, "download", true,
2124                 arvadostest.ActiveUserUUID, "", arvadostest.FooCollectionPDH, "foo")
2125 }
2126
2127 func (s *IntegrationSuite) TestUploadLoggingPermission(c *check.C) {
2128         for _, adminperm := range []bool{true, false} {
2129                 for _, userperm := range []bool{true, false} {
2130
2131                         arv := arvados.NewClientFromEnv()
2132                         arv.AuthToken = arvadostest.ActiveToken
2133
2134                         var coll arvados.Collection
2135                         err := arv.RequestAndDecode(&coll,
2136                                 "POST",
2137                                 "/arvados/v1/collections",
2138                                 nil,
2139                                 map[string]interface{}{
2140                                         "ensure_unique_name": true,
2141                                         "collection": map[string]interface{}{
2142                                                 "name": "test collection",
2143                                         },
2144                                 })
2145                         c.Assert(err, check.Equals, nil)
2146
2147                         u := mustParseURL("http://" + coll.UUID + ".keep-web.example/bar")
2148
2149                         s.handler.Cluster.Collections.WebDAVPermission.Admin.Upload = adminperm
2150                         s.handler.Cluster.Collections.WebDAVPermission.User.Upload = userperm
2151
2152                         // Test admin permission
2153                         req := &http.Request{
2154                                 Method:     "PUT",
2155                                 Host:       u.Host,
2156                                 URL:        u,
2157                                 RequestURI: u.RequestURI(),
2158                                 Header: http.Header{
2159                                         "Authorization": {"Bearer " + arvadostest.AdminToken},
2160                                 },
2161                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
2162                         }
2163                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", adminperm,
2164                                 arvadostest.AdminUserUUID, coll.UUID, "", "bar")
2165
2166                         // Test user permission
2167                         req = &http.Request{
2168                                 Method:     "PUT",
2169                                 Host:       u.Host,
2170                                 URL:        u,
2171                                 RequestURI: u.RequestURI(),
2172                                 Header: http.Header{
2173                                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
2174                                 },
2175                                 Body: io.NopCloser(bytes.NewReader([]byte("bar"))),
2176                         }
2177                         s.checkUploadDownloadRequest(c, req, http.StatusCreated, "upload", userperm,
2178                                 arvadostest.ActiveUserUUID, coll.UUID, "", "bar")
2179                 }
2180         }
2181 }
2182
2183 func (s *IntegrationSuite) TestConcurrentWrites(c *check.C) {
2184         s.handler.Cluster.Collections.WebDAVCache.TTL = arvados.Duration(time.Second * 2)
2185         lockTidyInterval = time.Second
2186         client := arvados.NewClientFromEnv()
2187         client.AuthToken = arvadostest.ActiveTokenV2
2188         // Start small, and increase concurrency (2^2, 4^2, ...)
2189         // only until hitting failure. Avoids unnecessarily long
2190         // failure reports.
2191         for n := 2; n < 16 && !c.Failed(); n = n * 2 {
2192                 c.Logf("%s: n=%d", c.TestName(), n)
2193
2194                 var coll arvados.Collection
2195                 err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, nil)
2196                 c.Assert(err, check.IsNil)
2197                 defer client.RequestAndDecode(&coll, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
2198
2199                 var wg sync.WaitGroup
2200                 for i := 0; i < n && !c.Failed(); i++ {
2201                         i := i
2202                         wg.Add(1)
2203                         go func() {
2204                                 defer wg.Done()
2205                                 u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
2206                                 resp := httptest.NewRecorder()
2207                                 req, err := http.NewRequest("MKCOL", u.String(), nil)
2208                                 c.Assert(err, check.IsNil)
2209                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2210                                 s.handler.ServeHTTP(resp, req)
2211                                 c.Assert(resp.Code, check.Equals, http.StatusCreated)
2212                                 for j := 0; j < n && !c.Failed(); j++ {
2213                                         j := j
2214                                         wg.Add(1)
2215                                         go func() {
2216                                                 defer wg.Done()
2217                                                 content := fmt.Sprintf("i=%d/j=%d", i, j)
2218                                                 u := mustParseURL("http://" + coll.UUID + ".collections.example.com/" + content)
2219
2220                                                 resp := httptest.NewRecorder()
2221                                                 req, err := http.NewRequest("PUT", u.String(), strings.NewReader(content))
2222                                                 c.Assert(err, check.IsNil)
2223                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2224                                                 s.handler.ServeHTTP(resp, req)
2225                                                 c.Check(resp.Code, check.Equals, http.StatusCreated)
2226
2227                                                 time.Sleep(time.Second)
2228                                                 resp = httptest.NewRecorder()
2229                                                 req, err = http.NewRequest("GET", u.String(), nil)
2230                                                 c.Assert(err, check.IsNil)
2231                                                 req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2232                                                 s.handler.ServeHTTP(resp, req)
2233                                                 c.Check(resp.Code, check.Equals, http.StatusOK)
2234                                                 c.Check(resp.Body.String(), check.Equals, content)
2235                                         }()
2236                                 }
2237                         }()
2238                 }
2239                 wg.Wait()
2240                 for i := 0; i < n; i++ {
2241                         u := mustParseURL(fmt.Sprintf("http://%s.collections.example.com/i=%d", coll.UUID, i))
2242                         resp := httptest.NewRecorder()
2243                         req, err := http.NewRequest("PROPFIND", u.String(), &bytes.Buffer{})
2244                         c.Assert(err, check.IsNil)
2245                         req.Header.Set("Authorization", "Bearer "+client.AuthToken)
2246                         s.handler.ServeHTTP(resp, req)
2247                         c.Assert(resp.Code, check.Equals, http.StatusMultiStatus)
2248                 }
2249         }
2250 }