Merge branch '10998-keepweb-block-cache-size'
[arvados.git] / services / keep-web / handler_test.go
index 040638623748f8aa57150b314886871703157287..1090151e1ae3fd872da91e6badc3ff277008eba8 100644 (file)
@@ -17,18 +17,48 @@ import (
        "regexp"
        "strings"
 
+       "git.curoverse.com/arvados.git/lib/config"
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "git.curoverse.com/arvados.git/sdk/go/arvadostest"
        "git.curoverse.com/arvados.git/sdk/go/auth"
+       "git.curoverse.com/arvados.git/sdk/go/keepclient"
        check "gopkg.in/check.v1"
 )
 
 var _ = check.Suite(&UnitSuite{})
 
-type UnitSuite struct{}
+type UnitSuite struct {
+       Config *arvados.Config
+}
+
+func (s *UnitSuite) SetUpTest(c *check.C) {
+       ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), nil)
+       ldr.Path = "-"
+       cfg, err := ldr.Load()
+       c.Assert(err, check.IsNil)
+       s.Config = cfg
+}
+
+func (s *UnitSuite) TestKeepClientBlockCache(c *check.C) {
+       cfg := newConfig(s.Config)
+       cfg.cluster.Collections.WebDAVCache.MaxBlockEntries = 42
+       h := handler{Config: cfg}
+       c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Not(check.Equals), cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
+       u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/t=" + arvadostest.ActiveToken + "/foo")
+       req := &http.Request{
+               Method:     "GET",
+               Host:       u.Host,
+               URL:        u,
+               RequestURI: u.RequestURI(),
+       }
+       resp := httptest.NewRecorder()
+       h.ServeHTTP(resp, req)
+       c.Check(resp.Code, check.Equals, http.StatusOK)
+       c.Check(keepclient.DefaultBlockCache.MaxBlocks, check.Equals, cfg.cluster.Collections.WebDAVCache.MaxBlockEntries)
+}
 
 func (s *UnitSuite) TestCORSPreflight(c *check.C) {
-       h := handler{Config: DefaultConfig()}
+       h := handler{Config: newConfig(s.Config)}
        u := mustParseURL("http://keep-web.example/c=" + arvadostest.FooCollection + "/foo")
        req := &http.Request{
                Method:     "OPTIONS",
@@ -78,8 +108,8 @@ func (s *UnitSuite) TestInvalidUUID(c *check.C) {
                        RequestURI: u.RequestURI(),
                }
                resp := httptest.NewRecorder()
-               cfg := DefaultConfig()
-               cfg.AnonymousTokens = []string{arvadostest.AnonymousToken}
+               cfg := newConfig(s.Config)
+               cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
                h := handler{Config: cfg}
                h.ServeHTTP(resp, req)
                c.Check(resp.Code, check.Equals, http.StatusNotFound)
@@ -338,7 +368,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenRequestAttachment(c *check
 }
 
 func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
-       s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+       s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
        resp := s.testVhostRedirectTokenToCookie(c, "GET",
                "download.example.com/by_id/"+arvadostest.FooCollection+"/foo",
                "?api_token="+arvadostest.ActiveToken,
@@ -351,7 +381,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenSiteFS(c *check.C) {
 }
 
 func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
-       s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+       s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
        resp := s.testVhostRedirectTokenToCookie(c, "GET",
                "download.example.com/c="+arvadostest.WazVersion1Collection+"/waz",
                "?api_token="+arvadostest.ActiveToken,
@@ -373,7 +403,7 @@ func (s *IntegrationSuite) TestPastCollectionVersionFileAccess(c *check.C) {
 }
 
 func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C) {
-       s.testServer.Config.TrustAllContent = true
+       s.testServer.Config.cluster.Collections.TrustAllContent = true
        s.testVhostRedirectTokenToCookie(c, "GET",
                "example.com/c="+arvadostest.FooCollection+"/foo",
                "?api_token="+arvadostest.ActiveToken,
@@ -385,7 +415,7 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenTrustAllContent(c *check.C
 }
 
 func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *check.C) {
-       s.testServer.Config.AttachmentOnlyHost = "example.com:1234"
+       s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com:1234"
 
        s.testVhostRedirectTokenToCookie(c, "GET",
                "example.com/c="+arvadostest.FooCollection+"/foo",
@@ -430,7 +460,7 @@ func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie404(c *check.C)
 }
 
 func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
-       s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
+       s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
        s.testVhostRedirectTokenToCookie(c, "GET",
                "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
                "",
@@ -442,7 +472,7 @@ func (s *IntegrationSuite) TestAnonymousTokenOK(c *check.C) {
 }
 
 func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
-       s.testServer.Config.AnonymousTokens = []string{"anonymousTokenConfiguredButInvalid"}
+       s.testServer.Config.cluster.Users.AnonymousUserToken = "anonymousTokenConfiguredButInvalid"
        s.testVhostRedirectTokenToCookie(c, "GET",
                "example.com/c="+arvadostest.HelloWorldCollection+"/Hello%20world.txt",
                "",
@@ -454,7 +484,7 @@ func (s *IntegrationSuite) TestAnonymousTokenError(c *check.C) {
 }
 
 func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
-       s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+       s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
 
        client := s.testServer.Config.Client
        client.AuthToken = arvadostest.ActiveToken
@@ -465,8 +495,12 @@ func (s *IntegrationSuite) TestSpecialCharsInPath(c *check.C) {
        f.Close()
        mtxt, err := fs.MarshalManifest(".")
        c.Assert(err, check.IsNil)
-       coll := arvados.Collection{ManifestText: mtxt}
-       err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", client.UpdateBody(coll), nil)
+       var coll arvados.Collection
+       err = client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+               "collection": map[string]string{
+                       "manifest_text": mtxt,
+               },
+       })
        c.Assert(err, check.IsNil)
 
        u, _ := url.Parse("http://download.example.com/c=" + coll.UUID + "/")
@@ -555,8 +589,18 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
        return resp
 }
 
-func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
-       s.testServer.Config.AttachmentOnlyHost = "download.example.com"
+func (s *IntegrationSuite) TestDirectoryListingWithAnonymousToken(c *check.C) {
+       s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+       s.testDirectoryListing(c)
+}
+
+func (s *IntegrationSuite) TestDirectoryListingWithNoAnonymousToken(c *check.C) {
+       s.testServer.Config.cluster.Users.AnonymousUserToken = ""
+       s.testDirectoryListing(c)
+}
+
+func (s *IntegrationSuite) testDirectoryListing(c *check.C) {
+       s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "download.example.com"
        authHeader := http.Header{
                "Authorization": {"OAuth2 " + arvadostest.ActiveToken},
        }
@@ -580,10 +624,12 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
                        cutDirs: 1,
                },
                {
-                       uri:     "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
-                       header:  authHeader,
-                       expect:  []string{"dir1/foo", "dir1/bar"},
-                       cutDirs: 2,
+                       // URLs of this form ignore authHeader, and
+                       // FooAndBarFilesInDirUUID isn't public, so
+                       // this returns 404.
+                       uri:    "download.example.com/collections/" + arvadostest.FooAndBarFilesInDirUUID + "/",
+                       header: authHeader,
+                       expect: nil,
                },
                {
                        uri:     "download.example.com/users/active/foo_file_in_dir/",
@@ -691,7 +737,7 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
                        cutDirs: 2,
                },
        } {
-               c.Logf("HTML: %q => %q", trial.uri, trial.expect)
+               comment := check.Commentf("HTML: %q => %q", trial.uri, trial.expect)
                resp := httptest.NewRecorder()
                u := mustParseURL("//" + trial.uri)
                req := &http.Request{
@@ -720,19 +766,19 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
                        s.testServer.Handler.ServeHTTP(resp, req)
                }
                if trial.redirect != "" {
-                       c.Check(req.URL.Path, check.Equals, trial.redirect)
+                       c.Check(req.URL.Path, check.Equals, trial.redirect, comment)
                }
                if trial.expect == nil {
-                       c.Check(resp.Code, check.Equals, http.StatusNotFound)
+                       c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
                } else {
-                       c.Check(resp.Code, check.Equals, http.StatusOK)
+                       c.Check(resp.Code, check.Equals, http.StatusOK, comment)
                        for _, e := range trial.expect {
-                               c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`)
+                               c.Check(resp.Body.String(), check.Matches, `(?ms).*href="./`+e+`".*`, comment)
                        }
-                       c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`)
+                       c.Check(resp.Body.String(), check.Matches, `(?ms).*--cut-dirs=`+fmt.Sprintf("%d", trial.cutDirs)+` .*`, comment)
                }
 
-               c.Logf("WebDAV: %q => %q", trial.uri, trial.expect)
+               comment = check.Commentf("WebDAV: %q => %q", trial.uri, trial.expect)
                req = &http.Request{
                        Method:     "OPTIONS",
                        Host:       u.Host,
@@ -744,9 +790,9 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
                resp = httptest.NewRecorder()
                s.testServer.Handler.ServeHTTP(resp, req)
                if trial.expect == nil {
-                       c.Check(resp.Code, check.Equals, http.StatusNotFound)
+                       c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
                } else {
-                       c.Check(resp.Code, check.Equals, http.StatusOK)
+                       c.Check(resp.Code, check.Equals, http.StatusOK, comment)
                }
 
                req = &http.Request{
@@ -760,11 +806,11 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
                resp = httptest.NewRecorder()
                s.testServer.Handler.ServeHTTP(resp, req)
                if trial.expect == nil {
-                       c.Check(resp.Code, check.Equals, http.StatusNotFound)
+                       c.Check(resp.Code, check.Equals, http.StatusNotFound, comment)
                } else {
-                       c.Check(resp.Code, check.Equals, http.StatusMultiStatus)
+                       c.Check(resp.Code, check.Equals, http.StatusMultiStatus, comment)
                        for _, e := range trial.expect {
-                               c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`)
+                               c.Check(resp.Body.String(), check.Matches, `(?ms).*<D:href>`+filepath.Join(u.Path, e)+`</D:href>.*`, comment)
                        }
                }
        }
@@ -773,17 +819,20 @@ func (s *IntegrationSuite) TestDirectoryListing(c *check.C) {
 func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
        arv := arvados.NewClientFromEnv()
        var newCollection arvados.Collection
-       err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", arv.UpdateBody(&arvados.Collection{
-               OwnerUUID:    arvadostest.ActiveUserUUID,
-               ManifestText: ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
-               Name:         "keep-web test collection",
-       }), map[string]bool{"ensure_unique_name": true})
+       err := arv.RequestAndDecode(&newCollection, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+               "collection": map[string]string{
+                       "owner_uuid":    arvadostest.ActiveUserUUID,
+                       "manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt 0:3:bar.txt\n",
+                       "name":          "keep-web test collection",
+               },
+               "ensure_unique_name": true,
+       })
        c.Assert(err, check.IsNil)
        defer arv.RequestAndDecode(&newCollection, "DELETE", "arvados/v1/collections/"+newCollection.UUID, nil, nil)
 
        var updated arvados.Collection
        for _, fnm := range []string{"foo.txt", "bar.txt"} {
-               s.testServer.Config.AttachmentOnlyHost = "example.com"
+               s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = "example.com"
                u, _ := url.Parse("http://example.com/c=" + newCollection.UUID + "/" + fnm)
                req := &http.Request{
                        Method:     "DELETE",
@@ -808,7 +857,7 @@ func (s *IntegrationSuite) TestDeleteLastFile(c *check.C) {
 }
 
 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
-       s.testServer.Config.ManagementToken = arvadostest.ManagementToken
+       s.testServer.Config.cluster.ManagementToken = arvadostest.ManagementToken
        authHeader := http.Header{
                "Authorization": {"Bearer " + arvadostest.ManagementToken},
        }