13 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
14 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
15 "git.curoverse.com/arvados.git/sdk/go/keepclient"
16 check "gopkg.in/check.v1"
19 var _ = check.Suite(&IntegrationSuite{})
21 // IntegrationSuite tests need an API server and a keep-web server
22 type IntegrationSuite struct {
26 func (s *IntegrationSuite) TestNoToken(c *check.C) {
27 for _, token := range []string{
31 hdr, body, _ := s.runCurl(c, token, "dl.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
32 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
33 c.Check(body, check.Equals, "")
36 hdr, body, _ = s.runCurl(c, token, "dl.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
37 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
38 c.Check(body, check.Equals, "")
41 hdr, body, _ = s.runCurl(c, token, "dl.example.com", "/bad-route")
42 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
43 c.Check(body, check.Equals, "")
47 // TODO: Move most cases to functional tests -- at least use Go's own
48 // http client instead of forking curl. Just leave enough of an
49 // integration test to assure that the documented way of invoking curl
50 // really works against the server.
51 func (s *IntegrationSuite) Test404(c *check.C) {
52 for _, uri := range []string{
59 "/collections/" + arvadostest.FooCollection,
60 "/collections/" + arvadostest.FooCollection + "/",
61 // Non-existent file in collection
62 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
63 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
64 // Non-existent collection
65 "/collections/" + arvadostest.NonexistentCollection,
66 "/collections/" + arvadostest.NonexistentCollection + "/",
67 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
68 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
70 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "dl.example.com", uri)
71 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
72 c.Check(body, check.Equals, "")
76 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
78 c.Skip("skipping 1GB integration test in short mode")
80 s.test100BlockFile(c, 10000000)
83 func (s *IntegrationSuite) Test300MBFile(c *check.C) {
84 s.test100BlockFile(c, 3000000)
87 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
88 testdata := make([]byte, blocksize)
89 for i := 0; i < blocksize; i++ {
90 testdata[i] = byte(' ')
92 arv, err := arvadosclient.MakeArvadosClient()
93 c.Assert(err, check.Equals, nil)
94 arv.ApiToken = arvadostest.ActiveToken
95 kc, err := keepclient.MakeKeepClient(&arv)
96 c.Assert(err, check.Equals, nil)
97 loc, _, err := kc.PutB(testdata[:])
98 c.Assert(err, check.Equals, nil)
100 for i := 0; i < 100; i++ {
101 mtext = mtext + " " + loc
103 mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
104 coll := map[string]interface{}{}
105 err = arv.Create("collections",
106 map[string]interface{}{
107 "collection": map[string]interface{}{
108 "name": fmt.Sprintf("testdata blocksize=%d", blocksize),
109 "manifest_text": mtext,
112 c.Assert(err, check.Equals, nil)
113 uuid := coll["uuid"].(string)
115 hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".dl.example.com", "/testdata.bin")
116 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
117 c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
118 c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
119 c.Check(size, check.Equals, int64(blocksize)*100)
122 type curlCase struct {
130 func (s *IntegrationSuite) Test200(c *check.C) {
131 anonymousTokens = []string{arvadostest.AnonymousToken}
132 arv, err := arvadosclient.MakeArvadosClient()
133 c.Assert(err, check.Equals, nil)
134 arv.ApiToken = arvadostest.ActiveToken
135 kc, err := keepclient.MakeKeepClient(&arv)
136 c.Assert(err, check.Equals, nil)
137 kc.PutB([]byte("Hello world\n"))
138 kc.PutB([]byte("foo"))
139 for _, spec := range []curlCase{
142 auth: arvadostest.ActiveToken,
143 host: arvadostest.FooCollection + "--dl.example.com",
145 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
148 host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".dl.example.com",
149 path: "/t=" + arvadostest.ActiveToken + "/foo",
150 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
153 path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
154 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
157 path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
158 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
161 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
162 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
165 auth: "tokensobogus",
166 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
167 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
170 auth: arvadostest.ActiveToken,
171 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
172 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
175 auth: arvadostest.AnonymousToken,
176 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
177 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
180 // Anonymously accessible user agreement
182 path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
183 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
186 host: arvadostest.HelloWorldCollection + ".dl.example.com",
187 path: "/Hello%20world.txt",
188 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
191 host: arvadostest.HelloWorldCollection + ".dl.example.com",
192 path: "/_/Hello%20world.txt",
193 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
196 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
197 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
200 auth: arvadostest.ActiveToken,
201 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
202 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
205 auth: arvadostest.SpectatorToken,
206 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
207 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
210 auth: arvadostest.SpectatorToken,
211 host: arvadostest.HelloWorldCollection + "--dl.example.com",
212 path: "/Hello%20world.txt",
213 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
216 auth: arvadostest.SpectatorToken,
217 path: "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
218 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
223 host = "dl.example.com"
225 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
226 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
227 if strings.HasSuffix(spec.path, ".txt") {
228 c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
229 // TODO: Check some types that aren't
230 // automatically detected by Go's http server
231 // by sniffing the content.
233 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
237 // Return header block and body.
238 func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
239 curlArgs := []string{"--silent", "--show-error", "--include"}
240 testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
241 curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
243 curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
245 curlArgs = append(curlArgs, args...)
246 curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
247 c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
248 cmd := exec.Command("curl", curlArgs...)
249 stdout, err := cmd.StdoutPipe()
250 c.Assert(err, check.Equals, nil)
251 cmd.Stderr = cmd.Stdout
253 buf := make([]byte, 2<<27)
254 n, err := io.ReadFull(stdout, buf)
255 // Discard (but measure size of) anything past 128 MiB.
257 if err == io.ErrUnexpectedEOF {
261 c.Assert(err, check.Equals, nil)
262 discarded, err = io.Copy(ioutil.Discard, stdout)
263 c.Assert(err, check.Equals, nil)
266 // Without "-f", curl exits 0 as long as it gets a valid HTTP
267 // response from the server, even if the response status
268 // indicates that the request failed. In our test suite, we
269 // always expect a valid HTTP response, and we parse the
270 // headers ourselves. If curl exits non-zero, our testing
271 // environment is broken.
272 c.Assert(err, check.Equals, nil)
273 hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
274 c.Assert(len(hdrsAndBody), check.Equals, 2)
276 bodyPart = hdrsAndBody[1]
277 bodySize = int64(len(bodyPart)) + discarded
281 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
282 arvadostest.StartAPI()
283 arvadostest.StartKeep()
286 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
287 arvadostest.StopKeep()
288 arvadostest.StopAPI()
291 func (s *IntegrationSuite) SetUpTest(c *check.C) {
292 arvadostest.ResetEnv()
293 s.testServer = &server{}
295 address = "127.0.0.1:0"
296 err = s.testServer.Start()
297 c.Assert(err, check.Equals, nil)
300 func (s *IntegrationSuite) TearDownTest(c *check.C) {
302 if s.testServer != nil {
303 err = s.testServer.Close()
305 c.Check(err, check.Equals, nil)
308 // Gocheck boilerplate
309 func Test(t *testing.T) {