1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
25 "git.arvados.org/arvados.git/lib/config"
26 "git.arvados.org/arvados.git/sdk/go/arvados"
27 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
28 "git.arvados.org/arvados.git/sdk/go/arvadostest"
29 "git.arvados.org/arvados.git/sdk/go/ctxlog"
30 "git.arvados.org/arvados.git/sdk/go/httpserver"
31 "git.arvados.org/arvados.git/sdk/go/keepclient"
32 check "gopkg.in/check.v1"
35 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
37 var _ = check.Suite(&IntegrationSuite{})
39 // IntegrationSuite tests need an API server and a keep-web server
40 type IntegrationSuite struct {
41 testServer *httptest.Server
45 func (s *IntegrationSuite) TestNoToken(c *check.C) {
46 for _, token := range []string{
50 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
51 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
52 c.Check(body, check.Equals, notFoundMessage+"\n")
55 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
56 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
57 c.Check(body, check.Equals, notFoundMessage+"\n")
60 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
61 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
62 c.Check(body, check.Equals, notFoundMessage+"\n")
66 // TODO: Move most cases to functional tests -- at least use Go's own
67 // http client instead of forking curl. Just leave enough of an
68 // integration test to assure that the documented way of invoking curl
69 // really works against the server.
70 func (s *IntegrationSuite) Test404(c *check.C) {
71 for _, uri := range []string{
72 // Routing errors (always 404 regardless of what's stored in Keep)
77 // Implicit/generated index is not implemented yet;
78 // until then, return 404.
79 "/collections/" + arvadostest.FooCollection,
80 "/collections/" + arvadostest.FooCollection + "/",
81 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
82 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
83 // Non-existent file in collection
84 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
85 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
86 // Non-existent collection
87 "/collections/" + arvadostest.NonexistentCollection,
88 "/collections/" + arvadostest.NonexistentCollection + "/",
89 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
90 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
92 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
93 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
95 c.Check(body, check.Equals, notFoundMessage+"\n")
100 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
102 c.Skip("skipping 1GB integration test in short mode")
104 s.test100BlockFile(c, 10000000)
107 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
110 s.test100BlockFile(c, 30000)
113 s.test100BlockFile(c, 3000000)
117 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
118 testdata := make([]byte, blocksize)
119 for i := 0; i < blocksize; i++ {
120 testdata[i] = byte(' ')
122 arv, err := arvadosclient.MakeArvadosClient()
123 c.Assert(err, check.Equals, nil)
124 arv.ApiToken = arvadostest.ActiveToken
125 kc, err := keepclient.MakeKeepClient(arv)
126 c.Assert(err, check.Equals, nil)
127 loc, _, err := kc.PutB(testdata[:])
128 c.Assert(err, check.Equals, nil)
130 for i := 0; i < 100; i++ {
131 mtext = mtext + " " + loc
133 mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
134 coll := map[string]interface{}{}
135 err = arv.Create("collections",
136 map[string]interface{}{
137 "collection": map[string]interface{}{
138 "name": fmt.Sprintf("testdata blocksize=%d", blocksize),
139 "manifest_text": mtext,
142 c.Assert(err, check.Equals, nil)
143 uuid := coll["uuid"].(string)
145 hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
146 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
147 c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
148 c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
149 c.Check(size, check.Equals, int64(blocksize)*100)
152 type curlCase struct {
159 func (s *IntegrationSuite) Test200(c *check.C) {
160 s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
161 for _, spec := range []curlCase{
164 auth: arvadostest.ActiveToken,
165 host: arvadostest.FooCollection + "--collections.example.com",
167 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
170 auth: arvadostest.ActiveToken,
171 host: arvadostest.FooCollection + ".collections.example.com",
173 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
176 host: strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
177 path: "/t=" + arvadostest.ActiveToken + "/foo",
178 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
181 path: "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
182 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
185 path: "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
186 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
189 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
190 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
193 auth: "tokensobogus",
194 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
195 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
198 auth: arvadostest.ActiveToken,
199 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
200 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
203 auth: arvadostest.AnonymousToken,
204 path: "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
205 dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
208 // Anonymously accessible data
210 path: "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
211 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
214 host: arvadostest.HelloWorldCollection + ".collections.example.com",
215 path: "/Hello%20world.txt",
216 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
219 host: arvadostest.HelloWorldCollection + ".collections.example.com",
220 path: "/_/Hello%20world.txt",
221 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
224 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
225 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
228 auth: arvadostest.ActiveToken,
229 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
230 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
233 auth: arvadostest.SpectatorToken,
234 path: "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
235 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
238 auth: arvadostest.SpectatorToken,
239 host: arvadostest.HelloWorldCollection + "--collections.example.com",
240 path: "/Hello%20world.txt",
241 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
244 auth: arvadostest.SpectatorToken,
245 path: "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
246 dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
251 host = "collections.example.com"
253 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
254 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
255 if strings.HasSuffix(spec.path, ".txt") {
256 c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
257 // TODO: Check some types that aren't
258 // automatically detected by Go's http server
259 // by sniffing the content.
261 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
265 // Return header block and body.
266 func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
267 curlArgs := []string{"--silent", "--show-error", "--include"}
268 testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
269 curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
270 if strings.Contains(auth, " ") {
271 // caller supplied entire Authorization header value
272 curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
273 } else if auth != "" {
274 // caller supplied Arvados token
275 curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
277 curlArgs = append(curlArgs, args...)
278 curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
279 c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
280 cmd := exec.Command("curl", curlArgs...)
281 stdout, err := cmd.StdoutPipe()
282 c.Assert(err, check.IsNil)
283 cmd.Stderr = os.Stderr
285 c.Assert(err, check.IsNil)
286 buf := make([]byte, 2<<27)
287 n, err := io.ReadFull(stdout, buf)
288 // Discard (but measure size of) anything past 128 MiB.
290 if err == io.ErrUnexpectedEOF {
293 c.Assert(err, check.IsNil)
294 discarded, err = io.Copy(ioutil.Discard, stdout)
295 c.Assert(err, check.IsNil)
298 // Without "-f", curl exits 0 as long as it gets a valid HTTP
299 // response from the server, even if the response status
300 // indicates that the request failed. In our test suite, we
301 // always expect a valid HTTP response, and we parse the
302 // headers ourselves. If curl exits non-zero, our testing
303 // environment is broken.
304 c.Assert(err, check.Equals, nil)
305 hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
306 c.Assert(len(hdrsAndBody), check.Equals, 2)
308 bodyPart = hdrsAndBody[1]
309 bodySize = int64(len(bodyPart)) + discarded
313 // Run a full-featured server, including the metrics/health routes
314 // that are added by service.Command.
315 func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
316 logbuf = &bytes.Buffer{}
317 cluster = *s.handler.Cluster
318 cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
319 cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
321 var configjson bytes.Buffer
322 json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
323 go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
324 for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
325 if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
326 srvaddr = "http://" + m[1]
336 // Ensure uploads can take longer than API.RequestTimeout.
338 // Currently, this works only by accident: service.Command cancels the
339 // request context as usual (there is no exemption), but
340 // webdav.Handler doesn't notice if the request context is cancelled
341 // while waiting to send or receive file data.
342 func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
343 s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
344 _, srvaddr, _ := s.runServer(c)
346 var coll arvados.Collection
347 arv, err := arvadosclient.MakeArvadosClient()
348 c.Assert(err, check.IsNil)
349 arv.ApiToken = arvadostest.ActiveTokenV2
350 err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
351 c.Assert(err, check.IsNil)
355 time.Sleep(time.Second)
356 pw.Write(make([]byte, 10000000))
359 req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
360 req.Host = coll.UUID + ".example"
361 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
362 resp, err := http.DefaultClient.Do(req)
363 c.Assert(err, check.IsNil)
364 c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
366 req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
367 req.Host = coll.UUID + ".example"
368 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
369 resp, err = http.DefaultClient.Do(req)
370 c.Assert(err, check.IsNil)
371 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
372 time.Sleep(time.Second)
373 body, err := ioutil.ReadAll(resp.Body)
374 c.Check(err, check.IsNil)
375 c.Check(len(body), check.Equals, 10000000)
378 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
379 cluster, srvaddr, _ := s.runServer(c)
380 req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
381 req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
382 resp, err := http.DefaultClient.Do(req)
383 c.Assert(err, check.IsNil)
384 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
385 body, _ := ioutil.ReadAll(resp.Body)
386 c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
389 func (s *IntegrationSuite) TestMetrics(c *check.C) {
390 cluster, srvaddr, _ := s.runServer(c)
392 req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
393 req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
394 _, err := http.DefaultClient.Do(req)
395 c.Assert(err, check.IsNil)
396 req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
397 req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
398 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
399 resp, err := http.DefaultClient.Do(req)
400 c.Assert(err, check.IsNil)
401 c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
402 for i := 0; i < 2; i++ {
403 req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
404 req.Host = arvadostest.FooCollection + ".example.com"
405 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
406 resp, err = http.DefaultClient.Do(req)
407 c.Assert(err, check.IsNil)
408 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
409 buf, _ := ioutil.ReadAll(resp.Body)
410 c.Check(buf, check.DeepEquals, []byte("foo"))
414 time.Sleep(metricsUpdateInterval * 2)
416 req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
417 req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
418 resp, err = http.DefaultClient.Do(req)
419 c.Assert(err, check.IsNil)
420 c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
422 req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
423 req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
424 req.Header.Set("Authorization", "Bearer badtoken")
425 resp, err = http.DefaultClient.Do(req)
426 c.Assert(err, check.IsNil)
427 c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
429 req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
430 req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
431 req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
432 resp, err = http.DefaultClient.Do(req)
433 c.Assert(err, check.IsNil)
434 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
435 type summary struct {
439 type counter struct {
459 json.NewDecoder(resp.Body).Decode(&ents)
460 summaries := map[string]summary{}
461 gauges := map[string]gauge{}
462 counters := map[string]counter{}
463 for _, e := range ents {
464 for _, m := range e.Metric {
465 labels := map[string]string{}
466 for _, lbl := range m.Label {
467 labels[lbl.Name] = lbl.Value
469 summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
470 counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
471 gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
474 c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
475 c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
476 c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
477 c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
478 c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
479 c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(2))
480 c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
481 c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
482 c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
483 // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
484 c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51))
486 // If the Host header indicates a collection, /metrics.json
487 // refers to a file in the collection -- the metrics handler
488 // must not intercept that route. Ditto health check paths.
489 for _, path := range []string{"/metrics.json", "/_health/ping"} {
490 c.Logf("path: %q", path)
491 req, _ = http.NewRequest("GET", srvaddr+path, nil)
492 req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
493 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
494 resp, err = http.DefaultClient.Do(req)
495 c.Assert(err, check.IsNil)
496 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
500 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
501 arvadostest.ResetDB(c)
502 arvadostest.StartKeep(2, true)
504 arv, err := arvadosclient.MakeArvadosClient()
505 c.Assert(err, check.Equals, nil)
506 arv.ApiToken = arvadostest.ActiveToken
507 kc, err := keepclient.MakeKeepClient(arv)
508 c.Assert(err, check.Equals, nil)
509 kc.PutB([]byte("Hello world\n"))
510 kc.PutB([]byte("foo"))
511 kc.PutB([]byte("foobar"))
512 kc.PutB([]byte("waz"))
515 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
516 arvadostest.StopKeep(2)
519 func (s *IntegrationSuite) SetUpTest(c *check.C) {
520 arvadostest.ResetEnv()
521 logger := ctxlog.TestLogger(c)
522 ldr := config.NewLoader(&bytes.Buffer{}, logger)
523 cfg, err := ldr.Load()
524 c.Assert(err, check.IsNil)
525 cluster, err := cfg.GetCluster("")
526 c.Assert(err, check.IsNil)
528 ctx := ctxlog.Context(context.Background(), logger)
530 s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, nil).(*handler)
531 s.testServer = httptest.NewUnstartedServer(
532 httpserver.AddRequestIDs(
533 httpserver.LogRequests(
535 s.testServer.Config.BaseContext = func(net.Listener) context.Context { return ctx }
538 cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
539 cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
542 func (s *IntegrationSuite) TearDownTest(c *check.C) {
543 if s.testServer != nil {
548 // Gocheck boilerplate
549 func Test(t *testing.T) {