1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
23 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
24 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
25 "git.curoverse.com/arvados.git/sdk/go/keepclient"
30 // Gocheck boilerplate
31 func Test(t *testing.T) {
35 // Gocheck boilerplate
36 var _ = Suite(&ServerRequiredSuite{})
38 // Tests that require the Keep server running
39 type ServerRequiredSuite struct{}
41 // Gocheck boilerplate
42 var _ = Suite(&NoKeepServerSuite{})
44 // Test with no keepserver to simulate errors
45 type NoKeepServerSuite struct{}
47 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
49 // Wait (up to 1 second) for keepproxy to listen on a port. This
50 // avoids a race condition where we hit a "connection refused" error
51 // because we start testing the proxy too soon.
52 func waitForListener() {
56 for i := 0; listener == nil && i < 10000; i += ms {
57 time.Sleep(ms * time.Millisecond)
60 log.Fatalf("Timed out waiting for listener to start")
64 func closeListener() {
70 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
71 arvadostest.StartAPI()
72 arvadostest.StartKeep(2, false)
75 func (s *ServerRequiredSuite) SetUpTest(c *C) {
76 arvadostest.ResetEnv()
79 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
80 arvadostest.StopKeep(2)
84 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
85 arvadostest.StartAPI()
86 // We need API to have some keep services listed, but the
87 // services themselves should be unresponsive.
88 arvadostest.StartKeep(2, false)
89 arvadostest.StopKeep(2)
92 func (s *NoKeepServerSuite) SetUpTest(c *C) {
93 arvadostest.ResetEnv()
96 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
100 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
101 args = append([]string{"keepproxy"}, args...)
102 os.Args = append(args, "-listen=:0")
107 arv, err := arvadosclient.MakeArvadosClient()
108 c.Assert(err, Equals, nil)
109 if bogusClientToken {
110 arv.ApiToken = "bogus-token"
112 kc := keepclient.New(arv)
113 sr := map[string]string{
114 TestProxyUUID: "http://" + listener.Addr().String(),
116 kc.SetServiceRoots(sr, sr, sr)
117 kc.Arvados.External = true
122 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
123 runProxy(c, nil, false)
124 defer closeListener()
126 req, err := http.NewRequest("POST",
127 "http://"+listener.Addr().String()+"/",
128 strings.NewReader("TestViaHeader"))
129 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
130 resp, err := (&http.Client{}).Do(req)
131 c.Assert(err, Equals, nil)
132 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
133 locator, err := ioutil.ReadAll(resp.Body)
134 c.Assert(err, Equals, nil)
137 req, err = http.NewRequest("GET",
138 "http://"+listener.Addr().String()+"/"+string(locator),
140 c.Assert(err, Equals, nil)
141 resp, err = (&http.Client{}).Do(req)
142 c.Assert(err, Equals, nil)
143 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
147 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
148 kc := runProxy(c, nil, false)
149 defer closeListener()
151 sr := map[string]string{
152 TestProxyUUID: "http://" + listener.Addr().String(),
154 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
156 content := []byte("TestLoopDetection")
157 _, _, err := kc.PutB(content)
158 c.Check(err, ErrorMatches, `.*loop detected.*`)
160 hash := fmt.Sprintf("%x", md5.Sum(content))
161 _, _, _, err = kc.Get(hash)
162 c.Check(err, ErrorMatches, `.*loop detected.*`)
165 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
166 kc := runProxy(c, nil, false)
167 defer closeListener()
169 content := []byte("TestDesiredReplicas")
170 hash := fmt.Sprintf("%x", md5.Sum(content))
172 for _, kc.Want_replicas = range []int{0, 1, 2} {
173 locator, rep, err := kc.PutB(content)
174 c.Check(err, Equals, nil)
175 c.Check(rep, Equals, kc.Want_replicas)
177 c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
182 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
183 kc := runProxy(c, nil, false)
184 defer closeListener()
186 content := []byte("TestPutWrongContentLength")
187 hash := fmt.Sprintf("%x", md5.Sum(content))
189 // If we use http.Client to send these requests to the network
190 // server we just started, the Go http library automatically
191 // fixes the invalid Content-Length header. In order to test
192 // our server behavior, we have to call the handler directly
193 // using an httptest.ResponseRecorder.
194 rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
196 type testcase struct {
201 for _, t := range []testcase{
202 {"1", http.StatusBadRequest},
203 {"", http.StatusLengthRequired},
204 {"-1", http.StatusLengthRequired},
205 {"abcdef", http.StatusLengthRequired},
207 req, err := http.NewRequest("PUT",
208 fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
209 bytes.NewReader(content))
211 req.Header.Set("Content-Length", t.sendLength)
212 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
213 req.Header.Set("Content-Type", "application/octet-stream")
215 resp := httptest.NewRecorder()
216 rtr.ServeHTTP(resp, req)
217 c.Check(resp.Code, Equals, t.expectStatus)
221 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
222 kc := runProxy(c, nil, false)
223 defer closeListener()
224 router.(*proxyHandler).timeout = time.Nanosecond
226 buf := make([]byte, 1<<20)
228 var wg sync.WaitGroup
229 for i := 0; i < 128; i++ {
236 done := make(chan bool)
243 case <-time.After(10 * time.Second):
248 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
249 kc := runProxy(c, nil, false)
250 defer closeListener()
252 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
256 _, _, err := kc.Ask(hash)
257 c.Check(err, Equals, keepclient.BlockNotFound)
258 log.Print("Finished Ask (expected BlockNotFound)")
262 reader, _, _, err := kc.Get(hash)
263 c.Check(reader, Equals, nil)
264 c.Check(err, Equals, keepclient.BlockNotFound)
265 log.Print("Finished Get (expected BlockNotFound)")
268 // Note in bug #5309 among other errors keepproxy would set
269 // Content-Length incorrectly on the 404 BlockNotFound response, this
270 // would result in a protocol violation that would prevent reuse of the
271 // connection, which would manifest by the next attempt to use the
272 // connection (in this case the PutB below) failing. So to test for
273 // that bug it's necessary to trigger an error response (such as
274 // BlockNotFound) and then do something else with the same httpClient
280 hash2, rep, err = kc.PutB([]byte("foo"))
281 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
282 c.Check(rep, Equals, 2)
283 c.Check(err, Equals, nil)
284 log.Print("Finished PutB (expected success)")
288 blocklen, _, err := kc.Ask(hash2)
289 c.Assert(err, Equals, nil)
290 c.Check(blocklen, Equals, int64(3))
291 log.Print("Finished Ask (expected success)")
295 reader, blocklen, _, err := kc.Get(hash2)
296 c.Assert(err, Equals, nil)
297 all, err := ioutil.ReadAll(reader)
298 c.Check(all, DeepEquals, []byte("foo"))
299 c.Check(blocklen, Equals, int64(3))
300 log.Print("Finished Get (expected success)")
306 hash2, rep, err = kc.PutB([]byte(""))
307 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
308 c.Check(rep, Equals, 2)
309 c.Check(err, Equals, nil)
310 log.Print("Finished PutB zero block")
314 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
315 c.Assert(err, Equals, nil)
316 all, err := ioutil.ReadAll(reader)
317 c.Check(all, DeepEquals, []byte(""))
318 c.Check(blocklen, Equals, int64(0))
319 log.Print("Finished Get zero block")
323 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
324 kc := runProxy(c, nil, true)
325 defer closeListener()
327 hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
330 _, _, err := kc.Ask(hash)
331 errNotFound, _ := err.(keepclient.ErrNotFound)
332 c.Check(errNotFound, NotNil)
333 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
338 hash2, rep, err := kc.PutB([]byte("bar"))
339 c.Check(hash2, Equals, "")
340 c.Check(rep, Equals, 0)
341 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
346 blocklen, _, err := kc.Ask(hash)
347 errNotFound, _ := err.(keepclient.ErrNotFound)
348 c.Check(errNotFound, NotNil)
349 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
350 c.Check(blocklen, Equals, int64(0))
355 _, blocklen, _, err := kc.Get(hash)
356 errNotFound, _ := err.(keepclient.ErrNotFound)
357 c.Check(errNotFound, NotNil)
358 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
359 c.Check(blocklen, Equals, int64(0))
364 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
365 kc := runProxy(c, []string{"-no-get"}, false)
366 defer closeListener()
368 hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
371 _, _, err := kc.Ask(hash)
372 errNotFound, _ := err.(keepclient.ErrNotFound)
373 c.Check(errNotFound, NotNil)
374 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
379 hash2, rep, err := kc.PutB([]byte("baz"))
380 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
381 c.Check(rep, Equals, 2)
382 c.Check(err, Equals, nil)
387 blocklen, _, err := kc.Ask(hash)
388 errNotFound, _ := err.(keepclient.ErrNotFound)
389 c.Check(errNotFound, NotNil)
390 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
391 c.Check(blocklen, Equals, int64(0))
396 _, blocklen, _, err := kc.Get(hash)
397 errNotFound, _ := err.(keepclient.ErrNotFound)
398 c.Check(errNotFound, NotNil)
399 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
400 c.Check(blocklen, Equals, int64(0))
405 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
406 kc := runProxy(c, []string{"-no-put"}, false)
407 defer closeListener()
409 hash2, rep, err := kc.PutB([]byte("quux"))
410 c.Check(hash2, Equals, "")
411 c.Check(rep, Equals, 0)
412 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
415 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
416 runProxy(c, nil, false)
417 defer closeListener()
420 client := http.Client{}
421 req, err := http.NewRequest("OPTIONS",
422 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
424 req.Header.Add("Access-Control-Request-Method", "PUT")
425 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
426 resp, err := client.Do(req)
427 c.Check(err, Equals, nil)
428 c.Check(resp.StatusCode, Equals, 200)
429 body, err := ioutil.ReadAll(resp.Body)
430 c.Check(string(body), Equals, "")
431 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
432 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
436 resp, err := http.Get(
437 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
438 c.Check(err, Equals, nil)
439 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
440 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
444 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
445 runProxy(c, nil, false)
446 defer closeListener()
449 client := http.Client{}
450 req, err := http.NewRequest("POST",
451 "http://"+listener.Addr().String()+"/",
452 strings.NewReader("qux"))
453 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
454 req.Header.Add("Content-Type", "application/octet-stream")
455 resp, err := client.Do(req)
456 c.Check(err, Equals, nil)
457 body, err := ioutil.ReadAll(resp.Body)
458 c.Check(err, Equals, nil)
459 c.Check(string(body), Matches,
460 fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
464 func (s *ServerRequiredSuite) TestStripHint(c *C) {
465 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
467 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
468 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
470 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
471 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
473 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
474 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
476 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
481 // Put one block, with 2 replicas
482 // With no prefix (expect the block locator, twice)
483 // With an existing prefix (expect the block locator, twice)
484 // With a valid but non-existing prefix (expect "\n")
485 // With an invalid prefix (expect error)
486 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
487 kc := runProxy(c, nil, false)
488 defer closeListener()
490 // Put "index-data" blocks
491 data := []byte("index-data")
492 hash := fmt.Sprintf("%x", md5.Sum(data))
494 hash2, rep, err := kc.PutB(data)
495 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
496 c.Check(rep, Equals, 2)
497 c.Check(err, Equals, nil)
499 reader, blocklen, _, err := kc.Get(hash)
500 c.Assert(err, Equals, nil)
501 c.Check(blocklen, Equals, int64(10))
502 all, err := ioutil.ReadAll(reader)
503 c.Check(all, DeepEquals, data)
505 // Put some more blocks
506 _, rep, err = kc.PutB([]byte("some-more-index-data"))
507 c.Check(err, Equals, nil)
509 kc.Arvados.ApiToken = arvadostest.DataManagerToken
512 for _, spec := range []struct {
517 {"", true, true}, // with no prefix
518 {hash[:3], true, false}, // with matching prefix
519 {"abcdef", false, false}, // with no such prefix
521 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
522 c.Assert(err, Equals, nil)
523 indexResp, err := ioutil.ReadAll(indexReader)
524 c.Assert(err, Equals, nil)
525 locators := strings.Split(string(indexResp), "\n")
528 for _, locator := range locators {
532 c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
533 if locator[:32] == hash {
539 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
540 c.Check(gotOther > 0, Equals, spec.expectOther)
543 // GetIndex with invalid prefix
544 _, err = kc.GetIndex(TestProxyUUID, "xyz")
545 c.Assert((err != nil), Equals, true)
548 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
549 kc := runProxy(c, nil, false)
550 defer closeListener()
553 hash, rep, err := kc.PutB([]byte("foo"))
554 c.Check(err, Equals, nil)
555 c.Check(rep, Equals, 2)
557 for _, token := range []string{
559 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
561 // Change token to given bad token
562 kc.Arvados.ApiToken = token
564 // Ask should result in error
565 _, _, err = kc.Ask(hash)
567 errNotFound, _ := err.(keepclient.ErrNotFound)
568 c.Check(errNotFound.Temporary(), Equals, false)
569 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
571 // Get should result in error
572 _, _, _, err = kc.Get(hash)
574 errNotFound, _ = err.(keepclient.ErrNotFound)
575 c.Check(errNotFound.Temporary(), Equals, false)
576 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
580 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
581 arv, err := arvadosclient.MakeArvadosClient()
582 c.Assert(err, Equals, nil)
584 // keepclient with no such keep server
585 kc := keepclient.New(arv)
586 locals := map[string]string{
587 TestProxyUUID: "http://localhost:12345",
589 kc.SetServiceRoots(locals, nil, nil)
591 // Ask should result in temporary connection refused error
592 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
593 _, _, err = kc.Ask(hash)
595 errNotFound, _ := err.(*keepclient.ErrNotFound)
596 c.Check(errNotFound.Temporary(), Equals, true)
597 c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
599 // Get should result in temporary connection refused error
600 _, _, _, err = kc.Get(hash)
602 errNotFound, _ = err.(*keepclient.ErrNotFound)
603 c.Check(errNotFound.Temporary(), Equals, true)
604 c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
607 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
608 kc := runProxy(c, nil, false)
609 defer closeListener()
611 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
612 for _, f := range []func() error{
614 _, _, err := kc.Ask(hash)
618 _, _, _, err := kc.Get(hash)
623 c.Assert(err, NotNil)
624 errNotFound, _ := err.(*keepclient.ErrNotFound)
625 c.Check(errNotFound.Temporary(), Equals, true)
626 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
630 func (s *ServerRequiredSuite) TestPing(c *C) {
631 kc := runProxy(c, nil, false)
632 defer closeListener()
634 rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
636 req, err := http.NewRequest("GET",
637 "http://"+listener.Addr().String()+"/_health/ping",
640 req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
642 resp := httptest.NewRecorder()
643 rtr.ServeHTTP(resp, req)
644 c.Check(resp.Code, Equals, 200)
645 c.Assert(strings.Contains(resp.Body.String(), `{"health":"OK"}`), Equals, true)