//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepweb
import (
"bytes"
+ "context"
"crypto/md5"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
+ "net/http/httptest"
"os"
"os/exec"
+ "regexp"
"strings"
"testing"
+ "time"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
+ "github.com/prometheus/client_golang/prometheus"
check "gopkg.in/check.v1"
)
// IntegrationSuite tests need an API server and a keep-web server
type IntegrationSuite struct {
- testServer *server
- ArvConfig *arvados.Config
+ testServer *httptest.Server
+ handler *handler
}
func (s *IntegrationSuite) TestNoToken(c *check.C) {
"",
"bogustoken",
} {
- hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
- c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, notFoundMessage+"\n")
+ hdr, body, _ := s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/c="+arvadostest.FooCollection+"/foo")
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`)
+ c.Check(strings.TrimSpace(body), check.Equals, unauthorizedMessage)
if token != "" {
- hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
+ hdr, body, _ = s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, notFoundMessage+"\n")
+ c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
}
- hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
+ hdr, body, _ = s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/bad-route")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, notFoundMessage+"\n")
+ c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
}
}
"/download",
"/collections",
"/collections/",
- // Implicit/generated index is not implemented yet;
- // until then, return 404.
- "/collections/" + arvadostest.FooCollection,
- "/collections/" + arvadostest.FooCollection + "/",
- "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
- "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
- // Non-existent file in collection
- "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
+ // Non-existent file/directory
+ "/c=" + arvadostest.FooCollection + "/theperthcountyconspiracy",
+ "/c=" + arvadostest.FooCollection + "/theperthcountyconspiracy/",
"/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
+ "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy/",
// Non-existent collection
- "/collections/" + arvadostest.NonexistentCollection,
- "/collections/" + arvadostest.NonexistentCollection + "/",
- "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
+ "/c=" + arvadostest.NonexistentCollection,
+ "/c=" + arvadostest.NonexistentCollection + "/",
+ "/c=" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
"/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
} {
- hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
+ hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, uri)
c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
if len(body) > 0 {
- c.Check(body, check.Equals, notFoundMessage+"\n")
+ c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
}
}
}
}
func (s *IntegrationSuite) Test200(c *check.C) {
- s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
+ s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
for _, spec := range []curlCase{
// My collection
{
}
// Return header block and body.
-func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
+func (s *IntegrationSuite) runCurl(c *check.C, auth, hostport, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
- testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
- curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
+ testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
+ host, port, _ := net.SplitHostPort(hostport)
+ if port == "" {
+ port = "80"
+ }
+ curlArgs = append(curlArgs, "--connect-to", host+":"+port+":"+testHost+":"+testPort)
if strings.Contains(auth, " ") {
// caller supplied entire Authorization header value
curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
}
curlArgs = append(curlArgs, args...)
- curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
+ curlArgs = append(curlArgs, "http://"+hostport+uri)
c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
cmd := exec.Command("curl", curlArgs...)
stdout, err := cmd.StdoutPipe()
return
}
+// Run a full-featured server, including the metrics/health routes
+// that are added by service.Command.
+func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
+ logbuf = &bytes.Buffer{}
+ cluster = *s.handler.Cluster
+ cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+ cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
+
+ var configjson bytes.Buffer
+ json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
+ go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
+ for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
+ if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
+ srvaddr = "http://" + m[1]
+ break
+ }
+ }
+ if srvaddr == "" {
+ c.Fatal("timed out")
+ }
+ return
+}
+
+// Ensure uploads can take longer than API.RequestTimeout.
+//
+// Currently, this works only by accident: service.Command cancels the
+// request context as usual (there is no exemption), but
+// webdav.Handler doesn't notice if the request context is cancelled
+// while waiting to send or receive file data.
+func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
+ s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
+ _, srvaddr, _ := s.runServer(c)
+
+ var coll arvados.Collection
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv.ApiToken = arvadostest.ActiveTokenV2
+ err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
+ c.Assert(err, check.IsNil)
+
+ pr, pw := io.Pipe()
+ go func() {
+ time.Sleep(time.Second)
+ pw.Write(make([]byte, 10000000))
+ pw.Close()
+ }()
+ req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
+ req.Host = coll.UUID + ".example"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
+
+ req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
+ req.Host = coll.UUID + ".example"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ time.Sleep(time.Second)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(len(body), check.Equals, 10000000)
+}
+
+func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
+ cluster, srvaddr, _ := s.runServer(c)
+ req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
+ req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ body, _ := ioutil.ReadAll(resp.Body)
+ c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
+}
+
func (s *IntegrationSuite) TestMetrics(c *check.C) {
- s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
- origin := "http://" + s.testServer.Addr
- req, _ := http.NewRequest("GET", origin+"/notfound", nil)
+ cluster, srvaddr, _ := s.runServer(c)
+
+ req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
_, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
- req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
resp, err := http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
for i := 0; i < 2; i++ {
- req, _ = http.NewRequest("GET", origin+"/foo", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
req.Host = arvadostest.FooCollection + ".example.com"
req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
resp, err = http.DefaultClient.Do(req)
resp.Body.Close()
}
- s.testServer.Config.Cache.updateGauges()
+ var coll arvados.Collection
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.IsNil)
+ arv.ApiToken = arvadostest.ActiveTokenV2
+ err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
+ c.Assert(err, check.IsNil)
+ defer arv.Delete("collections", coll.UUID, nil, nil)
+ for i := 0; i < 2; i++ {
+ size := 1 << (i * 12)
+ req, _ = http.NewRequest("PUT", srvaddr+"/zero-"+fmt.Sprintf("%d", size), bytes.NewReader(make([]byte, size)))
+ req.Host = coll.UUID + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
+ resp.Body.Close()
+ }
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ time.Sleep(metricsUpdateInterval * 2)
+
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer badtoken")
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
- c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
- c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(2))
- c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
- c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
- c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
- c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
- // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
- c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51))
+ c.Check(gauges["arvados_keepweb_sessions_cached_session_bytes//"].Value, check.Equals, float64(992))
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
- // must not intercept that route.
- req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
- req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
- req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ // must not intercept that route. Ditto health check paths.
+ for _, path := range []string{"/metrics.json", "/_health/ping"} {
+ c.Logf("path: %q", path)
+ req, _ = http.NewRequest("GET", srvaddr+path, nil)
+ req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ }
+
+ req, _ = http.NewRequest("GET", srvaddr+"/metrics", nil)
+ req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
resp, err = http.DefaultClient.Do(req)
c.Assert(err, check.IsNil)
- c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ allmetrics, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+
+ c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_apparent_backend_speed_bucket{size_range="0",le="\+Inf"} 4\n.*`)
+ c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_speed_bucket{size_range="0",le="\+Inf"} 4\n.*`)
+ c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_upload_speed_bucket{size_range="0",le="\+Inf"} 2\n.*`)
+ c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_upload_sync_delay_seconds_bucket{size_range="0",le="10"} 2\n.*`)
+
+ c.Logf("%s", allmetrics)
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
- arvadostest.StartAPI()
+ arvadostest.ResetDB(c)
arvadostest.StartKeep(2, true)
arv, err := arvadosclient.MakeArvadosClient()
func (s *IntegrationSuite) TearDownSuite(c *check.C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
- ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
- ldr.Path = "-"
- arvCfg, err := ldr.Load()
- c.Check(err, check.IsNil)
- cfg := newConfig(arvCfg)
+ logger := ctxlog.TestLogger(c)
+ ldr := config.NewLoader(&bytes.Buffer{}, logger)
+ cfg, err := ldr.Load()
c.Assert(err, check.IsNil)
- cfg.Client = arvados.Client{
- APIHost: testAPIHost,
- Insecure: true,
- }
- listen := "127.0.0.1:0"
- cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
- cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
- cfg.cluster.ManagementToken = arvadostest.ManagementToken
- cfg.cluster.SystemRootToken = arvadostest.SystemRootToken
- cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
- s.ArvConfig = arvCfg
- s.testServer = &server{Config: cfg}
- err = s.testServer.Start(ctxlog.TestLogger(c))
- c.Assert(err, check.Equals, nil)
+ cluster, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+
+ ctx := ctxlog.Context(context.Background(), logger)
+
+ s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, prometheus.NewRegistry()).(*handler)
+ s.testServer = httptest.NewUnstartedServer(
+ httpserver.AddRequestIDs(
+ httpserver.LogRequests(
+ s.handler)))
+ s.testServer.Config.BaseContext = func(net.Listener) context.Context { return ctx }
+ s.testServer.Start()
+
+ cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
+ cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
}
func (s *IntegrationSuite) TearDownTest(c *check.C) {
- var err error
if s.testServer != nil {
- err = s.testServer.Close()
+ s.testServer.Close()
}
- c.Check(err, check.Equals, nil)
}
// Gocheck boilerplate
projects / arvados.git / blobdiff