14 "git.curoverse.com/arvados.git/sdk/go/arvados"
15 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
16 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
17 "git.curoverse.com/arvados.git/sdk/go/keepclient"
18 check "gopkg.in/check.v1"
21 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
23 var _ = check.Suite(&IntegrationSuite{})
25 // IntegrationSuite tests need an API server and a keep-web server
26 type IntegrationSuite struct {
30 func (s *IntegrationSuite) TestNoToken(c *check.C) {
31 for _, token := range []string{
35 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
36 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
37 c.Check(body, check.Equals, "")
40 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
41 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
42 c.Check(body, check.Equals, "")
45 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
46 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
47 c.Check(body, check.Equals, "")
51 // TODO: Move most cases to functional tests -- at least use Go's own
52 // http client instead of forking curl. Just leave enough of an
53 // integration test to assure that the documented way of invoking curl
54 // really works against the server.
55 func (s *IntegrationSuite) Test404(c *check.C) {
56 for _, uri := range []string{
57 // Routing errors (always 404 regardless of what's stored in Keep)
63 // Implicit/generated index is not implemented yet;
64 // until then, return 404.
65 "/collections/" + arvadostest.FooCollection,
66 "/collections/" + arvadostest.FooCollection + "/",
67 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
68 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
69 // Non-existent file in collection
70 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
71 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
72 // Non-existent collection
73 "/collections/" + arvadostest.NonexistentCollection,
74 "/collections/" + arvadostest.NonexistentCollection + "/",
75 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
76 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
78 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
79 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
81 c.Check(body, check.Equals, "404 page not found\n")
86 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
88 c.Skip("skipping 1GB integration test in short mode")
90 s.test100BlockFile(c, 10000000)
93 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
96 s.test100BlockFile(c, 30000)
99 s.test100BlockFile(c, 3000000)
103 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
104 testdata := make([]byte, blocksize)
105 for i := 0; i < blocksize; i++ {
106 testdata[i] = byte(' ')
108 arv, err := arvadosclient.MakeArvadosClient()
109 c.Assert(err, check.Equals, nil)
110 arv.ApiToken = arvadostest.ActiveToken
111 kc, err := keepclient.MakeKeepClient(arv)
112 c.Assert(err, check.Equals, nil)
113 loc, _, err := kc.PutB(testdata[:])
114 c.Assert(err, check.Equals, nil)
116 for i := 0; i < 100; i++ {
117 mtext = mtext + " " + loc
119 mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
120 coll := map[string]interface{}{}
121 err = arv.Create("collections",
122 map[string]interface{}{
123 "collection": map[string]interface{}{
124 "name": fmt.Sprintf("testdata blocksize=%d", blocksize),
125 "manifest_text": mtext,
128 c.Assert(err, check.Equals, nil)
129 uuid := coll["uuid"].(string)
131 hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
132 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
133 c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
134 c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
135 c.Check(size, check.Equals, int64(blocksize)*100)
138 type curlCase struct {
145 func (s *IntegrationSuite) Test200(c *check.C) {
146 s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
147 for _, spec := range []curlCase{
150 auth: arvadostest.ActiveToken,
151 host: arvadostest.FooCollection + "--collections.example.com",
153 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
156 auth: arvadostest.ActiveToken,
157 host: arvadostest.FooCollection + ".collections.example.com",
159 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
162 host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
163 path: "/t=" + arvadostest.ActiveToken + "/foo",
164 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
167 path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
168 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
171 path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
172 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
175 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
176 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
179 auth: "tokensobogus",
180 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
181 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
184 auth: arvadostest.ActiveToken,
185 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
186 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
189 auth: arvadostest.AnonymousToken,
190 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
191 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
194 // Anonymously accessible data
196 path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
197 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
200 host: arvadostest.HelloWorldCollection + ".collections.example.com",
201 path: "/Hello%20world.txt",
202 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
205 host: arvadostest.HelloWorldCollection + ".collections.example.com",
206 path: "/_/Hello%20world.txt",
207 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
210 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
211 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
214 auth: arvadostest.ActiveToken,
215 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
216 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
219 auth: arvadostest.SpectatorToken,
220 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
221 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
224 auth: arvadostest.SpectatorToken,
225 host: arvadostest.HelloWorldCollection + "--collections.example.com",
226 path: "/Hello%20world.txt",
227 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
230 auth: arvadostest.SpectatorToken,
231 path: "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
232 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
237 host = "collections.example.com"
239 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
240 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
241 if strings.HasSuffix(spec.path, ".txt") {
242 c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
243 // TODO: Check some types that aren't
244 // automatically detected by Go's http server
245 // by sniffing the content.
247 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
251 // Return header block and body.
252 func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
253 curlArgs := []string{"--silent", "--show-error", "--include"}
254 testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
255 curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
257 curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
259 curlArgs = append(curlArgs, args...)
260 curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
261 c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
262 cmd := exec.Command("curl", curlArgs...)
263 stdout, err := cmd.StdoutPipe()
264 c.Assert(err, check.Equals, nil)
265 cmd.Stderr = cmd.Stdout
267 buf := make([]byte, 2<<27)
268 n, err := io.ReadFull(stdout, buf)
269 // Discard (but measure size of) anything past 128 MiB.
271 if err == io.ErrUnexpectedEOF {
275 c.Assert(err, check.Equals, nil)
276 discarded, err = io.Copy(ioutil.Discard, stdout)
277 c.Assert(err, check.Equals, nil)
280 // Without "-f", curl exits 0 as long as it gets a valid HTTP
281 // response from the server, even if the response status
282 // indicates that the request failed. In our test suite, we
283 // always expect a valid HTTP response, and we parse the
284 // headers ourselves. If curl exits non-zero, our testing
285 // environment is broken.
286 c.Assert(err, check.Equals, nil)
287 hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
288 c.Assert(len(hdrsAndBody), check.Equals, 2)
290 bodyPart = hdrsAndBody[1]
291 bodySize = int64(len(bodyPart)) + discarded
295 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
296 arvadostest.StartAPI()
297 arvadostest.StartKeep(2, true)
299 arv, err := arvadosclient.MakeArvadosClient()
300 c.Assert(err, check.Equals, nil)
301 arv.ApiToken = arvadostest.ActiveToken
302 kc, err := keepclient.MakeKeepClient(arv)
303 c.Assert(err, check.Equals, nil)
304 kc.PutB([]byte("Hello world\n"))
305 kc.PutB([]byte("foo"))
306 kc.PutB([]byte("foobar"))
309 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
310 arvadostest.StopKeep(2)
311 arvadostest.StopAPI()
314 func (s *IntegrationSuite) SetUpTest(c *check.C) {
315 arvadostest.ResetEnv()
316 cfg := DefaultConfig()
317 cfg.Client = arvados.Client{
318 APIHost: testAPIHost,
321 cfg.Listen = "127.0.0.1:0"
322 s.testServer = &server{Config: cfg}
323 err := s.testServer.Start()
324 c.Assert(err, check.Equals, nil)
327 func (s *IntegrationSuite) TearDownTest(c *check.C) {
329 if s.testServer != nil {
330 err = s.testServer.Close()
332 c.Check(err, check.Equals, nil)
335 // Gocheck boilerplate
336 func Test(t *testing.T) {