18947: Refactor keep-web as arvados-server command.
[arvados.git] / services / keep-web / server_test.go
index 0a38384f076fe7aba70f1cfe6940decb469e0dbe..dd8ce06172a01b3b90774f7985d8ab686f450af8 100644 (file)
@@ -1,26 +1,45 @@
-package main
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package keepweb
 
 import (
+       "bytes"
+       "context"
        "crypto/md5"
+       "encoding/json"
        "fmt"
        "io"
        "io/ioutil"
        "net"
+       "net/http"
+       "net/http/httptest"
+       "os"
        "os/exec"
+       "regexp"
        "strings"
        "testing"
+       "time"
 
-       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/arvadostest"
-       "git.curoverse.com/arvados.git/sdk/go/keepclient"
+       "git.arvados.org/arvados.git/lib/config"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "git.arvados.org/arvados.git/sdk/go/ctxlog"
+       "git.arvados.org/arvados.git/sdk/go/httpserver"
+       "git.arvados.org/arvados.git/sdk/go/keepclient"
        check "gopkg.in/check.v1"
 )
 
+var testAPIHost = os.Getenv("ARVADOS_API_HOST")
+
 var _ = check.Suite(&IntegrationSuite{})
 
 // IntegrationSuite tests need an API server and a keep-web server
 type IntegrationSuite struct {
-       testServer *server
+       testServer *httptest.Server
+       handler    *handler
 }
 
 func (s *IntegrationSuite) TestNoToken(c *check.C) {
@@ -30,17 +49,17 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
        } {
                hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
                c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
-               c.Check(body, check.Equals, "")
+               c.Check(body, check.Equals, notFoundMessage+"\n")
 
                if token != "" {
                        hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
                        c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
-                       c.Check(body, check.Equals, "")
+                       c.Check(body, check.Equals, notFoundMessage+"\n")
                }
 
                hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
                c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
-               c.Check(body, check.Equals, "")
+               c.Check(body, check.Equals, notFoundMessage+"\n")
        }
 }
 
@@ -50,14 +69,17 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
 // really works against the server.
 func (s *IntegrationSuite) Test404(c *check.C) {
        for _, uri := range []string{
-               // Routing errors
-               "/",
+               // Routing errors (always 404 regardless of what's stored in Keep)
                "/foo",
                "/download",
                "/collections",
                "/collections/",
+               // Implicit/generated index is not implemented yet;
+               // until then, return 404.
                "/collections/" + arvadostest.FooCollection,
                "/collections/" + arvadostest.FooCollection + "/",
+               "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
+               "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
                // Non-existent file in collection
                "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
                "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
@@ -69,7 +91,9 @@ func (s *IntegrationSuite) Test404(c *check.C) {
        } {
                hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
                c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
-               c.Check(body, check.Equals, "")
+               if len(body) > 0 {
+                       c.Check(body, check.Equals, notFoundMessage+"\n")
+               }
        }
 }
 
@@ -80,8 +104,14 @@ func (s *IntegrationSuite) Test1GBFile(c *check.C) {
        s.test100BlockFile(c, 10000000)
 }
 
-func (s *IntegrationSuite) Test300MBFile(c *check.C) {
-       s.test100BlockFile(c, 3000000)
+func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
+       if testing.Short() {
+               // 3 MB
+               s.test100BlockFile(c, 30000)
+       } else {
+               // 300 MB
+               s.test100BlockFile(c, 3000000)
+       }
 }
 
 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
@@ -92,7 +122,7 @@ func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
        arv, err := arvadosclient.MakeArvadosClient()
        c.Assert(err, check.Equals, nil)
        arv.ApiToken = arvadostest.ActiveToken
-       kc, err := keepclient.MakeKeepClient(&arv)
+       kc, err := keepclient.MakeKeepClient(arv)
        c.Assert(err, check.Equals, nil)
        loc, _, err := kc.PutB(testdata[:])
        c.Assert(err, check.Equals, nil)
@@ -120,7 +150,6 @@ func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 }
 
 type curlCase struct {
-       id      string
        auth    string
        host    string
        path    string
@@ -128,7 +157,7 @@ type curlCase struct {
 }
 
 func (s *IntegrationSuite) Test200(c *check.C) {
-       anonymousTokens = []string{arvadostest.AnonymousToken}
+       s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
        for _, spec := range []curlCase{
                // My collection
                {
@@ -138,16 +167,22 @@ func (s *IntegrationSuite) Test200(c *check.C) {
                        dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
                },
                {
-                       host:    strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
+                       auth:    arvadostest.ActiveToken,
+                       host:    arvadostest.FooCollection + ".collections.example.com",
+                       path:    "/foo",
+                       dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
+               },
+               {
+                       host:    strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
                        path:    "/t=" + arvadostest.ActiveToken + "/foo",
                        dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
                },
                {
-                       path:    "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
+                       path:    "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
                        dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
                },
                {
-                       path:    "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
+                       path:    "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
                        dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
                },
                {
@@ -170,7 +205,7 @@ func (s *IntegrationSuite) Test200(c *check.C) {
                        dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
                },
 
-               // Anonymously accessible user agreement
+               // Anonymously accessible data
                {
                        path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
                        dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
@@ -228,32 +263,36 @@ func (s *IntegrationSuite) Test200(c *check.C) {
 }
 
 // Return header block and body.
-func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
+func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
        curlArgs := []string{"--silent", "--show-error", "--include"}
-       testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
+       testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
        curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
-       if token != "" {
-               curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+       if strings.Contains(auth, " ") {
+               // caller supplied entire Authorization header value
+               curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
+       } else if auth != "" {
+               // caller supplied Arvados token
+               curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
        }
        curlArgs = append(curlArgs, args...)
        curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
        c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
        cmd := exec.Command("curl", curlArgs...)
        stdout, err := cmd.StdoutPipe()
-       c.Assert(err, check.Equals, nil)
-       cmd.Stderr = cmd.Stdout
-       go cmd.Start()
+       c.Assert(err, check.IsNil)
+       cmd.Stderr = os.Stderr
+       err = cmd.Start()
+       c.Assert(err, check.IsNil)
        buf := make([]byte, 2<<27)
        n, err := io.ReadFull(stdout, buf)
        // Discard (but measure size of) anything past 128 MiB.
        var discarded int64
        if err == io.ErrUnexpectedEOF {
-               err = nil
                buf = buf[:n]
        } else {
-               c.Assert(err, check.Equals, nil)
+               c.Assert(err, check.IsNil)
                discarded, err = io.Copy(ioutil.Discard, stdout)
-               c.Assert(err, check.Equals, nil)
+               c.Assert(err, check.IsNil)
        }
        err = cmd.Wait()
        // Without "-f", curl exits 0 as long as it gets a valid HTTP
@@ -271,40 +310,239 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...
        return
 }
 
+// Run a full-featured server, including the metrics/health routes
+// that are added by service.Command.
+func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
+       logbuf = &bytes.Buffer{}
+       cluster = *s.handler.Cluster
+       cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+       cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+
+       var configjson bytes.Buffer
+       json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
+       go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
+       for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
+               if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
+                       srvaddr = "http://" + m[1]
+                       break
+               }
+       }
+       if srvaddr == "" {
+               c.Fatal("timed out")
+       }
+       return
+}
+
+// Ensure uploads can take longer than API.RequestTimeout.
+//
+// Currently, this works only by accident: service.Command cancels the
+// request context as usual (there is no exemption), but
+// webdav.Handler doesn't notice if the request context is cancelled
+// while waiting to send or receive file data.
+func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
+       s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
+       _, srvaddr, _ := s.runServer(c)
+
+       var coll arvados.Collection
+       arv, err := arvadosclient.MakeArvadosClient()
+       c.Assert(err, check.IsNil)
+       arv.ApiToken = arvadostest.ActiveTokenV2
+       err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
+       c.Assert(err, check.IsNil)
+
+       pr, pw := io.Pipe()
+       go func() {
+               time.Sleep(time.Second)
+               pw.Write(make([]byte, 10000000))
+               pw.Close()
+       }()
+       req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
+       req.Host = coll.UUID + ".example"
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+       resp, err := http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
+
+       req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
+       req.Host = coll.UUID + ".example"
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+       resp, err = http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       time.Sleep(time.Second)
+       body, err := ioutil.ReadAll(resp.Body)
+       c.Check(err, check.IsNil)
+       c.Check(len(body), check.Equals, 10000000)
+}
+
+func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
+       cluster, srvaddr, _ := s.runServer(c)
+       req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
+       req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
+       resp, err := http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       body, _ := ioutil.ReadAll(resp.Body)
+       c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
+}
+
+func (s *IntegrationSuite) TestMetrics(c *check.C) {
+       cluster, srvaddr, _ := s.runServer(c)
+
+       req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
+       req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+       _, err := http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
+       req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+       resp, err := http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
+       for i := 0; i < 2; i++ {
+               req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
+               req.Host = arvadostest.FooCollection + ".example.com"
+               req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+               resp, err = http.DefaultClient.Do(req)
+               c.Assert(err, check.IsNil)
+               c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+               buf, _ := ioutil.ReadAll(resp.Body)
+               c.Check(buf, check.DeepEquals, []byte("foo"))
+               resp.Body.Close()
+       }
+
+       time.Sleep(metricsUpdateInterval * 2)
+
+       req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+       req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+       resp, err = http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
+
+       req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+       req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+       req.Header.Set("Authorization", "Bearer badtoken")
+       resp, err = http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
+
+       req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+       req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+       req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
+       resp, err = http.DefaultClient.Do(req)
+       c.Assert(err, check.IsNil)
+       c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+       type summary struct {
+               SampleCount string
+               SampleSum   float64
+       }
+       type counter struct {
+               Value int64
+       }
+       type gauge struct {
+               Value float64
+       }
+       var ents []struct {
+               Name   string
+               Help   string
+               Type   string
+               Metric []struct {
+                       Label []struct {
+                               Name  string
+                               Value string
+                       }
+                       Counter counter
+                       Gauge   gauge
+                       Summary summary
+               }
+       }
+       json.NewDecoder(resp.Body).Decode(&ents)
+       summaries := map[string]summary{}
+       gauges := map[string]gauge{}
+       counters := map[string]counter{}
+       for _, e := range ents {
+               for _, m := range e.Metric {
+                       labels := map[string]string{}
+                       for _, lbl := range m.Label {
+                               labels[lbl.Name] = lbl.Value
+                       }
+                       summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+                       counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
+                       gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
+               }
+       }
+       c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
+       c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
+       c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
+       c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+       c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
+       c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(2))
+       c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
+       c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
+       c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
+       // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
+       c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51))
+
+       // If the Host header indicates a collection, /metrics.json
+       // refers to a file in the collection -- the metrics handler
+       // must not intercept that route. Ditto health check paths.
+       for _, path := range []string{"/metrics.json", "/_health/ping"} {
+               c.Logf("path: %q", path)
+               req, _ = http.NewRequest("GET", srvaddr+path, nil)
+               req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
+               req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+               resp, err = http.DefaultClient.Do(req)
+               c.Assert(err, check.IsNil)
+               c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+       }
+}
+
 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
-       arvadostest.StartAPI()
-       arvadostest.StartKeep()
+       arvadostest.ResetDB(c)
+       arvadostest.StartKeep(2, true)
 
        arv, err := arvadosclient.MakeArvadosClient()
        c.Assert(err, check.Equals, nil)
        arv.ApiToken = arvadostest.ActiveToken
-       kc, err := keepclient.MakeKeepClient(&arv)
+       kc, err := keepclient.MakeKeepClient(arv)
        c.Assert(err, check.Equals, nil)
        kc.PutB([]byte("Hello world\n"))
        kc.PutB([]byte("foo"))
        kc.PutB([]byte("foobar"))
+       kc.PutB([]byte("waz"))
 }
 
 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
-       arvadostest.StopKeep()
-       arvadostest.StopAPI()
+       arvadostest.StopKeep(2)
 }
 
 func (s *IntegrationSuite) SetUpTest(c *check.C) {
        arvadostest.ResetEnv()
-       s.testServer = &server{}
-       var err error
-       address = "127.0.0.1:0"
-       err = s.testServer.Start()
-       c.Assert(err, check.Equals, nil)
+       logger := ctxlog.TestLogger(c)
+       ldr := config.NewLoader(&bytes.Buffer{}, logger)
+       cfg, err := ldr.Load()
+       c.Assert(err, check.IsNil)
+       cluster, err := cfg.GetCluster("")
+       c.Assert(err, check.IsNil)
+
+       ctx := ctxlog.Context(context.Background(), logger)
+
+       s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, nil).(*handler)
+       s.testServer = httptest.NewUnstartedServer(
+               httpserver.AddRequestIDs(
+                       httpserver.LogRequests(
+                               s.handler)))
+       s.testServer.Config.BaseContext = func(net.Listener) context.Context { return ctx }
+       s.testServer.Start()
+
+       cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
+       cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
 }
 
 func (s *IntegrationSuite) TearDownTest(c *check.C) {
-       var err error
        if s.testServer != nil {
-               err = s.testServer.Close()
+               s.testServer.Close()
        }
-       c.Check(err, check.Equals, nil)
 }
 
 // Gocheck boilerplate