X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/11890d971215ec44b7abb1e24ad748aefa1061a7..3ed14dc61e8d39d748d0e7c25d12e4e5c45e09a8:/services/keep-web/server_test.go diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go index a2a5754380..21fa8a4b5c 100644 --- a/services/keep-web/server_test.go +++ b/services/keep-web/server_test.go @@ -1,3 +1,7 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( @@ -6,16 +10,20 @@ import ( "io" "io/ioutil" "net" + "os" "os/exec" "strings" "testing" + "git.curoverse.com/arvados.git/sdk/go/arvados" "git.curoverse.com/arvados.git/sdk/go/arvadosclient" "git.curoverse.com/arvados.git/sdk/go/arvadostest" "git.curoverse.com/arvados.git/sdk/go/keepclient" check "gopkg.in/check.v1" ) +var testAPIHost = os.Getenv("ARVADOS_API_HOST") + var _ = check.Suite(&IntegrationSuite{}) // IntegrationSuite tests need an API server and a keep-web server @@ -28,17 +36,17 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) { "", "bogustoken", } { - hdr, body, _ := s.runCurl(c, token, "dl.example.com", "/collections/"+arvadostest.FooCollection+"/foo") - c.Check(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`) + hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo") + c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`) c.Check(body, check.Equals, "") if token != "" { - hdr, body, _ = s.runCurl(c, token, "dl.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo") + hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo") c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`) c.Check(body, check.Equals, "") } - hdr, body, _ = s.runCurl(c, token, "dl.example.com", "/bad-route") + hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route") c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`) c.Check(body, check.Equals, "") } @@ -50,14 +58,18 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) { // really works against the server. func (s *IntegrationSuite) Test404(c *check.C) { for _, uri := range []string{ - // Routing errors + // Routing errors (always 404 regardless of what's stored in Keep) "/", "/foo", "/download", "/collections", "/collections/", + // Implicit/generated index is not implemented yet; + // until then, return 404. "/collections/" + arvadostest.FooCollection, "/collections/" + arvadostest.FooCollection + "/", + "/collections/" + arvadostest.FooBarDirCollection + "/dir1", + "/collections/" + arvadostest.FooBarDirCollection + "/dir1/", // Non-existent file in collection "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy", "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy", @@ -67,9 +79,11 @@ func (s *IntegrationSuite) Test404(c *check.C) { "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy", "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy", } { - hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "dl.example.com", uri) + hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri) c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*") - c.Check(body, check.Equals, "") + if len(body) > 0 { + c.Check(body, check.Equals, "404 page not found\n") + } } } @@ -80,8 +94,14 @@ func (s *IntegrationSuite) Test1GBFile(c *check.C) { s.test100BlockFile(c, 10000000) } -func (s *IntegrationSuite) Test300MBFile(c *check.C) { - s.test100BlockFile(c, 3000000) +func (s *IntegrationSuite) Test100BlockFile(c *check.C) { + if testing.Short() { + // 3 MB + s.test100BlockFile(c, 30000) + } else { + // 300 MB + s.test100BlockFile(c, 3000000) + } } func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) { @@ -92,7 +112,7 @@ func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) { arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.Equals, nil) arv.ApiToken = arvadostest.ActiveToken - kc, err := keepclient.MakeKeepClient(&arv) + kc, err := keepclient.MakeKeepClient(arv) c.Assert(err, check.Equals, nil) loc, _, err := kc.PutB(testdata[:]) c.Assert(err, check.Equals, nil) @@ -105,51 +125,130 @@ func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) { err = arv.Create("collections", map[string]interface{}{ "collection": map[string]interface{}{ - "name": fmt.Sprintf("testdata blocksize=%d", blocksize), + "name": fmt.Sprintf("testdata blocksize=%d", blocksize), "manifest_text": mtext, }, }, &coll) c.Assert(err, check.Equals, nil) uuid := coll["uuid"].(string) - hdr, body, size := s.runCurl(c, arv.ApiToken, uuid + ".dl.example.com", "/testdata.bin") + hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin") c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`) c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`) c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234]) c.Check(size, check.Equals, int64(blocksize)*100) } +type curlCase struct { + auth string + host string + path string + dataMD5 string +} + func (s *IntegrationSuite) Test200(c *check.C) { - anonymousTokens = []string{arvadostest.AnonymousToken} - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, check.Equals, nil) - arv.ApiToken = arvadostest.ActiveToken - kc, err := keepclient.MakeKeepClient(&arv) - c.Assert(err, check.Equals, nil) - kc.PutB([]byte("Hello world\n")) - kc.PutB([]byte("foo")) - for _, spec := range [][]string{ + s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken} + for _, spec := range []curlCase{ // My collection - {arvadostest.ActiveToken, "/collections/" + arvadostest.FooCollection + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"}, - {"", "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"}, - {"tokensobogus", "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"}, - {arvadostest.ActiveToken, "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"}, - {arvadostest.AnonymousToken, "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", "acbd18db4cc2f85cedef654fccc4a4d8"}, - // Anonymously accessible user agreement. - {"", "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"}, - {arvadostest.ActiveToken, "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"}, - {arvadostest.SpectatorToken, "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"}, - {arvadostest.SpectatorToken, "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt", "f0ef7081e1539ac00ef5b761b4fb01b3"}, + { + auth: arvadostest.ActiveToken, + host: arvadostest.FooCollection + "--collections.example.com", + path: "/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + { + auth: arvadostest.ActiveToken, + host: arvadostest.FooCollection + ".collections.example.com", + path: "/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + { + host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com", + path: "/t=" + arvadostest.ActiveToken + "/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + { + path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + { + path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + { + path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + { + auth: "tokensobogus", + path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + { + auth: arvadostest.ActiveToken, + path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + { + auth: arvadostest.AnonymousToken, + path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo", + dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", + }, + + // Anonymously accessible data + { + path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", + dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3", + }, + { + host: arvadostest.HelloWorldCollection + ".collections.example.com", + path: "/Hello%20world.txt", + dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3", + }, + { + host: arvadostest.HelloWorldCollection + ".collections.example.com", + path: "/_/Hello%20world.txt", + dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3", + }, + { + path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", + dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3", + }, + { + auth: arvadostest.ActiveToken, + path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", + dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3", + }, + { + auth: arvadostest.SpectatorToken, + path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt", + dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3", + }, + { + auth: arvadostest.SpectatorToken, + host: arvadostest.HelloWorldCollection + "--collections.example.com", + path: "/Hello%20world.txt", + dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3", + }, + { + auth: arvadostest.SpectatorToken, + path: "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt", + dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3", + }, } { - hdr, body, _ := s.runCurl(c, spec[0], "dl.example.com", spec[1]) + host := spec.host + if host == "" { + host = "collections.example.com" + } + hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path) c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`) - if strings.HasSuffix(spec[1], ".txt") { + if strings.HasSuffix(spec.path, ".txt") { c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`) // TODO: Check some types that aren't // automatically detected by Go's http server // by sniffing the content. } - c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec[2]) + c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5) } } @@ -157,7 +256,7 @@ func (s *IntegrationSuite) Test200(c *check.C) { func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) { curlArgs := []string{"--silent", "--show-error", "--include"} testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr) - curlArgs = append(curlArgs, "--resolve", host + ":" + testPort + ":" + testHost) + curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost) if token != "" { curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token) } @@ -199,20 +298,33 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ... func (s *IntegrationSuite) SetUpSuite(c *check.C) { arvadostest.StartAPI() - arvadostest.StartKeep() + arvadostest.StartKeep(2, true) + + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, check.Equals, nil) + arv.ApiToken = arvadostest.ActiveToken + kc, err := keepclient.MakeKeepClient(arv) + c.Assert(err, check.Equals, nil) + kc.PutB([]byte("Hello world\n")) + kc.PutB([]byte("foo")) + kc.PutB([]byte("foobar")) } func (s *IntegrationSuite) TearDownSuite(c *check.C) { - arvadostest.StopKeep() + arvadostest.StopKeep(2) arvadostest.StopAPI() } func (s *IntegrationSuite) SetUpTest(c *check.C) { arvadostest.ResetEnv() - s.testServer = &server{} - var err error - address = "127.0.0.1:0" - err = s.testServer.Start() + cfg := DefaultConfig() + cfg.Client = arvados.Client{ + APIHost: testAPIHost, + Insecure: true, + } + cfg.Listen = "127.0.0.1:0" + s.testServer = &server{Config: cfg} + err := s.testServer.Start() c.Assert(err, check.Equals, nil) }