Add 'build/' from commit '555b039609a3c8700c27767c255fdfe00eb42063'
[arvados.git] / services / keep-web / server_test.go
1 package main
2
3 import (
4         "crypto/md5"
5         "fmt"
6         "io"
7         "io/ioutil"
8         "net"
9         "os/exec"
10         "strings"
11         "testing"
12
13         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
14         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
15         "git.curoverse.com/arvados.git/sdk/go/keepclient"
16         check "gopkg.in/check.v1"
17 )
18
19 var _ = check.Suite(&IntegrationSuite{})
20
21 // IntegrationSuite tests need an API server and a keep-web server
22 type IntegrationSuite struct {
23         testServer *server
24 }
25
26 func (s *IntegrationSuite) TestNoToken(c *check.C) {
27         for _, token := range []string{
28                 "",
29                 "bogustoken",
30         } {
31                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
32                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
33                 c.Check(body, check.Equals, "")
34
35                 if token != "" {
36                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
37                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
38                         c.Check(body, check.Equals, "")
39                 }
40
41                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
42                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
43                 c.Check(body, check.Equals, "")
44         }
45 }
46
47 // TODO: Move most cases to functional tests -- at least use Go's own
48 // http client instead of forking curl. Just leave enough of an
49 // integration test to assure that the documented way of invoking curl
50 // really works against the server.
51 func (s *IntegrationSuite) Test404(c *check.C) {
52         for _, uri := range []string{
53                 // Routing errors (always 404 regardless of what's stored in Keep)
54                 "/",
55                 "/foo",
56                 "/download",
57                 "/collections",
58                 "/collections/",
59                 // Implicit/generated index is not implemented yet;
60                 // until then, return 404.
61                 "/collections/" + arvadostest.FooCollection,
62                 "/collections/" + arvadostest.FooCollection + "/",
63                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
64                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
65                 // Non-existent file in collection
66                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
67                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
68                 // Non-existent collection
69                 "/collections/" + arvadostest.NonexistentCollection,
70                 "/collections/" + arvadostest.NonexistentCollection + "/",
71                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
72                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
73         } {
74                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
75                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
76                 c.Check(body, check.Equals, "")
77         }
78 }
79
80 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
81         if testing.Short() {
82                 c.Skip("skipping 1GB integration test in short mode")
83         }
84         s.test100BlockFile(c, 10000000)
85 }
86
87 func (s *IntegrationSuite) Test300MBFile(c *check.C) {
88         s.test100BlockFile(c, 3000000)
89 }
90
91 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
92         testdata := make([]byte, blocksize)
93         for i := 0; i < blocksize; i++ {
94                 testdata[i] = byte(' ')
95         }
96         arv, err := arvadosclient.MakeArvadosClient()
97         c.Assert(err, check.Equals, nil)
98         arv.ApiToken = arvadostest.ActiveToken
99         kc, err := keepclient.MakeKeepClient(&arv)
100         c.Assert(err, check.Equals, nil)
101         loc, _, err := kc.PutB(testdata[:])
102         c.Assert(err, check.Equals, nil)
103         mtext := "."
104         for i := 0; i < 100; i++ {
105                 mtext = mtext + " " + loc
106         }
107         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
108         coll := map[string]interface{}{}
109         err = arv.Create("collections",
110                 map[string]interface{}{
111                         "collection": map[string]interface{}{
112                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
113                                 "manifest_text": mtext,
114                         },
115                 }, &coll)
116         c.Assert(err, check.Equals, nil)
117         uuid := coll["uuid"].(string)
118
119         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
120         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
121         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
122         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
123         c.Check(size, check.Equals, int64(blocksize)*100)
124 }
125
126 type curlCase struct {
127         auth    string
128         host    string
129         path    string
130         dataMD5 string
131 }
132
133 func (s *IntegrationSuite) Test200(c *check.C) {
134         anonymousTokens = []string{arvadostest.AnonymousToken}
135         for _, spec := range []curlCase{
136                 // My collection
137                 {
138                         auth:    arvadostest.ActiveToken,
139                         host:    arvadostest.FooCollection + "--collections.example.com",
140                         path:    "/foo",
141                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
142                 },
143                 {
144                         auth:    arvadostest.ActiveToken,
145                         host:    arvadostest.FooCollection + ".collections.example.com",
146                         path:    "/foo",
147                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
148                 },
149                 {
150                         host:    strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
151                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
152                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
153                 },
154                 {
155                         path:    "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
156                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
157                 },
158                 {
159                         path:    "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
160                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
161                 },
162                 {
163                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
164                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
165                 },
166                 {
167                         auth:    "tokensobogus",
168                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
169                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
170                 },
171                 {
172                         auth:    arvadostest.ActiveToken,
173                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
174                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
175                 },
176                 {
177                         auth:    arvadostest.AnonymousToken,
178                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
179                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
180                 },
181
182                 // Anonymously accessible data
183                 {
184                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
185                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
186                 },
187                 {
188                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
189                         path:    "/Hello%20world.txt",
190                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
191                 },
192                 {
193                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
194                         path:    "/_/Hello%20world.txt",
195                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
196                 },
197                 {
198                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
199                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
200                 },
201                 {
202                         auth:    arvadostest.ActiveToken,
203                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
204                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
205                 },
206                 {
207                         auth:    arvadostest.SpectatorToken,
208                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
209                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
210                 },
211                 {
212                         auth:    arvadostest.SpectatorToken,
213                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
214                         path:    "/Hello%20world.txt",
215                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
216                 },
217                 {
218                         auth:    arvadostest.SpectatorToken,
219                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
220                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
221                 },
222         } {
223                 host := spec.host
224                 if host == "" {
225                         host = "collections.example.com"
226                 }
227                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
228                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
229                 if strings.HasSuffix(spec.path, ".txt") {
230                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
231                         // TODO: Check some types that aren't
232                         // automatically detected by Go's http server
233                         // by sniffing the content.
234                 }
235                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
236         }
237 }
238
239 // Return header block and body.
240 func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
241         curlArgs := []string{"--silent", "--show-error", "--include"}
242         testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
243         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
244         if token != "" {
245                 curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
246         }
247         curlArgs = append(curlArgs, args...)
248         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
249         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
250         cmd := exec.Command("curl", curlArgs...)
251         stdout, err := cmd.StdoutPipe()
252         c.Assert(err, check.Equals, nil)
253         cmd.Stderr = cmd.Stdout
254         go cmd.Start()
255         buf := make([]byte, 2<<27)
256         n, err := io.ReadFull(stdout, buf)
257         // Discard (but measure size of) anything past 128 MiB.
258         var discarded int64
259         if err == io.ErrUnexpectedEOF {
260                 err = nil
261                 buf = buf[:n]
262         } else {
263                 c.Assert(err, check.Equals, nil)
264                 discarded, err = io.Copy(ioutil.Discard, stdout)
265                 c.Assert(err, check.Equals, nil)
266         }
267         err = cmd.Wait()
268         // Without "-f", curl exits 0 as long as it gets a valid HTTP
269         // response from the server, even if the response status
270         // indicates that the request failed. In our test suite, we
271         // always expect a valid HTTP response, and we parse the
272         // headers ourselves. If curl exits non-zero, our testing
273         // environment is broken.
274         c.Assert(err, check.Equals, nil)
275         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
276         c.Assert(len(hdrsAndBody), check.Equals, 2)
277         hdr = hdrsAndBody[0]
278         bodyPart = hdrsAndBody[1]
279         bodySize = int64(len(bodyPart)) + discarded
280         return
281 }
282
283 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
284         arvadostest.StartAPI()
285         arvadostest.StartKeep(2, true)
286
287         arv, err := arvadosclient.MakeArvadosClient()
288         c.Assert(err, check.Equals, nil)
289         arv.ApiToken = arvadostest.ActiveToken
290         kc, err := keepclient.MakeKeepClient(&arv)
291         c.Assert(err, check.Equals, nil)
292         kc.PutB([]byte("Hello world\n"))
293         kc.PutB([]byte("foo"))
294         kc.PutB([]byte("foobar"))
295 }
296
297 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
298         arvadostest.StopKeep(2)
299         arvadostest.StopAPI()
300 }
301
302 func (s *IntegrationSuite) SetUpTest(c *check.C) {
303         arvadostest.ResetEnv()
304         s.testServer = &server{}
305         var err error
306         address = "127.0.0.1:0"
307         err = s.testServer.Start()
308         c.Assert(err, check.Equals, nil)
309 }
310
311 func (s *IntegrationSuite) TearDownTest(c *check.C) {
312         var err error
313         if s.testServer != nil {
314                 err = s.testServer.Close()
315         }
316         c.Check(err, check.Equals, nil)
317 }
318
319 // Gocheck boilerplate
320 func Test(t *testing.T) {
321         check.TestingT(t)
322 }