+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package main
import (
"crypto/md5"
+ "encoding/json"
"fmt"
"io"
"io/ioutil"
"net"
+ "net/http"
+ "os"
"os/exec"
"strings"
"testing"
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
+var testAPIHost = os.Getenv("ARVADOS_API_HOST")
+
var _ = check.Suite(&IntegrationSuite{})
// IntegrationSuite tests need an API server and a keep-web server
"",
"bogustoken",
} {
- hdr, body, _ := s.runCurl(c, token, "dl.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
+ hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
c.Check(body, check.Equals, "")
if token != "" {
- hdr, body, _ = s.runCurl(c, token, "dl.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
+ hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
c.Check(body, check.Equals, "")
}
- hdr, body, _ = s.runCurl(c, token, "dl.example.com", "/bad-route")
+ hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
c.Check(body, check.Equals, "")
}
// really works against the server.
func (s *IntegrationSuite) Test404(c *check.C) {
for _, uri := range []string{
- // Routing errors
- "/",
+ // Routing errors (always 404 regardless of what's stored in Keep)
"/foo",
"/download",
"/collections",
"/collections/",
+ // Implicit/generated index is not implemented yet;
+ // until then, return 404.
"/collections/" + arvadostest.FooCollection,
"/collections/" + arvadostest.FooCollection + "/",
+ "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
+ "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
// Non-existent file in collection
"/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
"/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
"/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
"/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
} {
- hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "dl.example.com", uri)
+ hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
- c.Check(body, check.Equals, "")
+ if len(body) > 0 {
+ c.Check(body, check.Equals, "404 page not found\n")
+ }
}
}
s.test100BlockFile(c, 10000000)
}
-func (s *IntegrationSuite) Test300MBFile(c *check.C) {
- s.test100BlockFile(c, 3000000)
+func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
+ if testing.Short() {
+ // 3 MB
+ s.test100BlockFile(c, 30000)
+ } else {
+ // 300 MB
+ s.test100BlockFile(c, 3000000)
+ }
}
func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, check.Equals, nil)
arv.ApiToken = arvadostest.ActiveToken
- kc, err := keepclient.MakeKeepClient(&arv)
+ kc, err := keepclient.MakeKeepClient(arv)
c.Assert(err, check.Equals, nil)
loc, _, err := kc.PutB(testdata[:])
c.Assert(err, check.Equals, nil)
err = arv.Create("collections",
map[string]interface{}{
"collection": map[string]interface{}{
- "name": fmt.Sprintf("testdata blocksize=%d", blocksize),
+ "name": fmt.Sprintf("testdata blocksize=%d", blocksize),
"manifest_text": mtext,
},
}, &coll)
c.Assert(err, check.Equals, nil)
uuid := coll["uuid"].(string)
- hdr, body, size := s.runCurl(c, arv.ApiToken, uuid + ".dl.example.com", "/testdata.bin")
+ hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
}
type curlCase struct {
- id string
auth string
host string
path string
}
func (s *IntegrationSuite) Test200(c *check.C) {
- anonymousTokens = []string{arvadostest.AnonymousToken}
- arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, check.Equals, nil)
- arv.ApiToken = arvadostest.ActiveToken
- kc, err := keepclient.MakeKeepClient(&arv)
- c.Assert(err, check.Equals, nil)
- kc.PutB([]byte("Hello world\n"))
- kc.PutB([]byte("foo"))
+ s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
for _, spec := range []curlCase{
// My collection
{
- auth: arvadostest.ActiveToken,
- host: arvadostest.FooCollection + "--dl.example.com",
- path: "/foo",
+ auth: arvadostest.ActiveToken,
+ host: arvadostest.FooCollection + "--collections.example.com",
+ path: "/foo",
+ dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
+ },
+ {
+ auth: arvadostest.ActiveToken,
+ host: arvadostest.FooCollection + ".collections.example.com",
+ path: "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".dl.example.com",
- path: "/t=" + arvadostest.ActiveToken + "/foo",
+ host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
+ path: "/t=" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
+ path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
+ path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
+ path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- auth: "tokensobogus",
- path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
+ auth: "tokensobogus",
+ path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- auth: arvadostest.ActiveToken,
- path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
+ auth: arvadostest.ActiveToken,
+ path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
{
- auth: arvadostest.AnonymousToken,
- path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
+ auth: arvadostest.AnonymousToken,
+ path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
},
- // Anonymously accessible user agreement
+ // Anonymously accessible data
{
- path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- host: arvadostest.HelloWorldCollection + ".dl.example.com",
- path: "/Hello%20world.txt",
+ host: arvadostest.HelloWorldCollection + ".collections.example.com",
+ path: "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- host: arvadostest.HelloWorldCollection + ".dl.example.com",
- path: "/_/Hello%20world.txt",
+ host: arvadostest.HelloWorldCollection + ".collections.example.com",
+ path: "/_/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- auth: arvadostest.ActiveToken,
- path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ auth: arvadostest.ActiveToken,
+ path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- auth: arvadostest.SpectatorToken,
- path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
+ auth: arvadostest.SpectatorToken,
+ path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- auth: arvadostest.SpectatorToken,
- host: arvadostest.HelloWorldCollection + "--dl.example.com",
- path: "/Hello%20world.txt",
+ auth: arvadostest.SpectatorToken,
+ host: arvadostest.HelloWorldCollection + "--collections.example.com",
+ path: "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
{
- auth: arvadostest.SpectatorToken,
- path: "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
+ auth: arvadostest.SpectatorToken,
+ path: "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
},
} {
host := spec.host
if host == "" {
- host = "dl.example.com"
+ host = "collections.example.com"
}
hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
- curlArgs = append(curlArgs, "--resolve", host + ":" + testPort + ":" + testHost)
+ curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
if token != "" {
curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
}
// Discard (but measure size of) anything past 128 MiB.
var discarded int64
if err == io.ErrUnexpectedEOF {
- err = nil
buf = buf[:n]
} else {
c.Assert(err, check.Equals, nil)
return
}
+func (s *IntegrationSuite) TestMetrics(c *check.C) {
+ origin := "http://" + s.testServer.Addr
+ req, _ := http.NewRequest("GET", origin+"/notfound", nil)
+ _, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ for i := 0; i < 2; i++ {
+ req, _ = http.NewRequest("GET", origin+"/foo", nil)
+ req.Host = arvadostest.FooCollection + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ buf, _ := ioutil.ReadAll(resp.Body)
+ c.Check(buf, check.DeepEquals, []byte("foo"))
+ resp.Body.Close()
+ }
+
+ s.testServer.Config.Cache.updateGauges()
+
+ req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
+
+ req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req.Header.Set("Authorization", "Bearer badtoken")
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
+
+ req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ type summary struct {
+ SampleCount string `json:"sample_count"`
+ SampleSum float64 `json:"sample_sum"`
+ Quantile []struct {
+ Quantile float64
+ Value float64
+ }
+ }
+ type counter struct {
+ Value int64
+ }
+ type gauge struct {
+ Value float64
+ }
+ var ents []struct {
+ Name string
+ Help string
+ Type string
+ Metric []struct {
+ Label []struct {
+ Name string
+ Value string
+ }
+ Counter counter
+ Gauge gauge
+ Summary summary
+ }
+ }
+ json.NewDecoder(resp.Body).Decode(&ents)
+ summaries := map[string]summary{}
+ gauges := map[string]gauge{}
+ counters := map[string]counter{}
+ for _, e := range ents {
+ for _, m := range e.Metric {
+ labels := map[string]string{}
+ for _, lbl := range m.Label {
+ labels[lbl.Name] = lbl.Value
+ }
+ summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
+ counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
+ gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
+ }
+ }
+ c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
+ c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
+ c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
+ c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
+ c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
+ c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
+ // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
+ c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
+
+ // If the Host header indicates a collection, /metrics.json
+ // refers to a file in the collection -- the metrics handler
+ // must not intercept that route.
+ req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
+ req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
+ req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+ resp, err = http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
+}
+
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
arvadostest.StartAPI()
- arvadostest.StartKeep()
+ arvadostest.StartKeep(2, true)
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+ kc.PutB([]byte("Hello world\n"))
+ kc.PutB([]byte("foo"))
+ kc.PutB([]byte("foobar"))
+ kc.PutB([]byte("waz"))
}
func (s *IntegrationSuite) TearDownSuite(c *check.C) {
- arvadostest.StopKeep()
+ arvadostest.StopKeep(2)
arvadostest.StopAPI()
}
func (s *IntegrationSuite) SetUpTest(c *check.C) {
arvadostest.ResetEnv()
- s.testServer = &server{}
- var err error
- address = "127.0.0.1:0"
- err = s.testServer.Start()
+ cfg := DefaultConfig()
+ cfg.Client = arvados.Client{
+ APIHost: testAPIHost,
+ Insecure: true,
+ }
+ cfg.Listen = "127.0.0.1:0"
+ cfg.ManagementToken = arvadostest.ManagementToken
+ s.testServer = &server{Config: cfg}
+ err := s.testServer.Start()
c.Assert(err, check.Equals, nil)
}