1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
21 "git.curoverse.com/arvados.git/lib/config"
22 "git.curoverse.com/arvados.git/sdk/go/arvados"
23 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
24 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
25 "git.curoverse.com/arvados.git/sdk/go/keepclient"
26 log "github.com/sirupsen/logrus"
31 // Gocheck boilerplate
32 func Test(t *testing.T) {
36 // Gocheck boilerplate
37 var _ = Suite(&ServerRequiredSuite{})
39 // Tests that require the Keep server running
40 type ServerRequiredSuite struct{}
42 // Gocheck boilerplate
43 var _ = Suite(&NoKeepServerSuite{})
45 // Test with no keepserver to simulate errors
46 type NoKeepServerSuite struct{}
48 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
50 // Wait (up to 1 second) for keepproxy to listen on a port. This
51 // avoids a race condition where we hit a "connection refused" error
52 // because we start testing the proxy too soon.
53 func waitForListener() {
57 for i := 0; listener == nil && i < 10000; i += ms {
58 time.Sleep(ms * time.Millisecond)
61 panic("Timed out waiting for listener to start")
65 func closeListener() {
71 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
72 arvadostest.StartAPI()
73 arvadostest.StartKeep(2, false)
76 func (s *ServerRequiredSuite) SetUpTest(c *C) {
77 arvadostest.ResetEnv()
80 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
81 arvadostest.StopKeep(2)
85 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
86 arvadostest.StartAPI()
87 // We need API to have some keep services listed, but the
88 // services themselves should be unresponsive.
89 arvadostest.StartKeep(2, false)
90 arvadostest.StopKeep(2)
93 func (s *NoKeepServerSuite) SetUpTest(c *C) {
94 arvadostest.ResetEnv()
97 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
101 func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
102 cfg, err := config.NewLoader(nil, nil).Load()
103 c.Assert(err, Equals, nil)
104 cluster, err := cfg.GetCluster("")
105 c.Assert(err, Equals, nil)
107 cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
111 run(log.New(), cluster)
112 defer closeListener()
116 client := arvados.NewClientFromEnv()
117 arv, err := arvadosclient.New(client)
118 c.Assert(err, Equals, nil)
119 if bogusClientToken {
120 arv.ApiToken = "bogus-token"
122 kc := keepclient.New(arv)
123 sr := map[string]string{
124 TestProxyUUID: "http://" + listener.Addr().String(),
126 kc.SetServiceRoots(sr, sr, sr)
127 kc.Arvados.External = true
132 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
134 defer closeListener()
136 req, err := http.NewRequest("POST",
137 "http://"+listener.Addr().String()+"/",
138 strings.NewReader("TestViaHeader"))
139 c.Assert(err, Equals, nil)
140 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
141 resp, err := (&http.Client{}).Do(req)
142 c.Assert(err, Equals, nil)
143 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
144 locator, err := ioutil.ReadAll(resp.Body)
145 c.Assert(err, Equals, nil)
148 req, err = http.NewRequest("GET",
149 "http://"+listener.Addr().String()+"/"+string(locator),
151 c.Assert(err, Equals, nil)
152 resp, err = (&http.Client{}).Do(req)
153 c.Assert(err, Equals, nil)
154 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
158 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
159 kc := runProxy(c, false)
160 defer closeListener()
162 sr := map[string]string{
163 TestProxyUUID: "http://" + listener.Addr().String(),
165 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
167 content := []byte("TestLoopDetection")
168 _, _, err := kc.PutB(content)
169 c.Check(err, ErrorMatches, `.*loop detected.*`)
171 hash := fmt.Sprintf("%x", md5.Sum(content))
172 _, _, _, err = kc.Get(hash)
173 c.Check(err, ErrorMatches, `.*loop detected.*`)
176 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
177 kc := runProxy(c, false)
178 defer closeListener()
180 // Set up fake keepstore to record request headers
182 ts := httptest.NewServer(http.HandlerFunc(
183 func(w http.ResponseWriter, r *http.Request) {
185 http.Error(w, "Error", http.StatusInternalServerError)
189 // Point keepproxy router's keepclient to the fake keepstore
190 sr := map[string]string{
191 TestProxyUUID: ts.URL,
193 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
195 // Set up client to ask for storage classes to keepproxy
196 kc.StorageClasses = []string{"secure"}
197 content := []byte("Very important data")
198 _, _, err := kc.PutB(content)
200 c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
203 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
204 kc := runProxy(c, false)
205 defer closeListener()
207 content := []byte("TestDesiredReplicas")
208 hash := fmt.Sprintf("%x", md5.Sum(content))
210 for _, kc.Want_replicas = range []int{0, 1, 2} {
211 locator, rep, err := kc.PutB(content)
212 c.Check(err, Equals, nil)
213 c.Check(rep, Equals, kc.Want_replicas)
215 c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
220 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
221 kc := runProxy(c, false)
222 defer closeListener()
224 content := []byte("TestPutWrongContentLength")
225 hash := fmt.Sprintf("%x", md5.Sum(content))
227 // If we use http.Client to send these requests to the network
228 // server we just started, the Go http library automatically
229 // fixes the invalid Content-Length header. In order to test
230 // our server behavior, we have to call the handler directly
231 // using an httptest.ResponseRecorder.
232 rtr := MakeRESTRouter(kc, 10*time.Second, "")
234 type testcase struct {
239 for _, t := range []testcase{
240 {"1", http.StatusBadRequest},
241 {"", http.StatusLengthRequired},
242 {"-1", http.StatusLengthRequired},
243 {"abcdef", http.StatusLengthRequired},
245 req, err := http.NewRequest("PUT",
246 fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
247 bytes.NewReader(content))
249 req.Header.Set("Content-Length", t.sendLength)
250 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
251 req.Header.Set("Content-Type", "application/octet-stream")
253 resp := httptest.NewRecorder()
254 rtr.ServeHTTP(resp, req)
255 c.Check(resp.Code, Equals, t.expectStatus)
259 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
260 kc := runProxy(c, false)
261 defer closeListener()
262 router.(*proxyHandler).timeout = time.Nanosecond
264 buf := make([]byte, 1<<20)
266 var wg sync.WaitGroup
267 for i := 0; i < 128; i++ {
274 done := make(chan bool)
281 case <-time.After(10 * time.Second):
286 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
287 kc := runProxy(c, false)
288 defer closeListener()
290 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
294 _, _, err := kc.Ask(hash)
295 c.Check(err, Equals, keepclient.BlockNotFound)
296 c.Log("Finished Ask (expected BlockNotFound)")
300 reader, _, _, err := kc.Get(hash)
301 c.Check(reader, Equals, nil)
302 c.Check(err, Equals, keepclient.BlockNotFound)
303 c.Log("Finished Get (expected BlockNotFound)")
306 // Note in bug #5309 among other errors keepproxy would set
307 // Content-Length incorrectly on the 404 BlockNotFound response, this
308 // would result in a protocol violation that would prevent reuse of the
309 // connection, which would manifest by the next attempt to use the
310 // connection (in this case the PutB below) failing. So to test for
311 // that bug it's necessary to trigger an error response (such as
312 // BlockNotFound) and then do something else with the same httpClient
318 hash2, rep, err = kc.PutB([]byte("foo"))
319 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
320 c.Check(rep, Equals, 2)
321 c.Check(err, Equals, nil)
322 c.Log("Finished PutB (expected success)")
326 blocklen, _, err := kc.Ask(hash2)
327 c.Assert(err, Equals, nil)
328 c.Check(blocklen, Equals, int64(3))
329 c.Log("Finished Ask (expected success)")
333 reader, blocklen, _, err := kc.Get(hash2)
334 c.Assert(err, Equals, nil)
335 all, err := ioutil.ReadAll(reader)
337 c.Check(all, DeepEquals, []byte("foo"))
338 c.Check(blocklen, Equals, int64(3))
339 c.Log("Finished Get (expected success)")
345 hash2, rep, err = kc.PutB([]byte(""))
346 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
347 c.Check(rep, Equals, 2)
348 c.Check(err, Equals, nil)
349 c.Log("Finished PutB zero block")
353 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
354 c.Assert(err, Equals, nil)
355 all, err := ioutil.ReadAll(reader)
357 c.Check(all, DeepEquals, []byte(""))
358 c.Check(blocklen, Equals, int64(0))
359 c.Log("Finished Get zero block")
363 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
364 kc := runProxy(c, true)
365 defer closeListener()
367 hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
369 _, _, err := kc.Ask(hash)
370 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
372 hash2, rep, err := kc.PutB([]byte("bar"))
373 c.Check(hash2, Equals, "")
374 c.Check(rep, Equals, 0)
375 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
377 blocklen, _, err := kc.Ask(hash)
378 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
379 c.Check(err, ErrorMatches, ".*not found.*")
380 c.Check(blocklen, Equals, int64(0))
382 _, blocklen, _, err = kc.Get(hash)
383 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
384 c.Check(err, ErrorMatches, ".*not found.*")
385 c.Check(blocklen, Equals, int64(0))
389 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
391 defer closeListener()
394 client := http.Client{}
395 req, err := http.NewRequest("OPTIONS",
396 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
399 req.Header.Add("Access-Control-Request-Method", "PUT")
400 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
401 resp, err := client.Do(req)
402 c.Check(err, Equals, nil)
403 c.Check(resp.StatusCode, Equals, 200)
404 body, err := ioutil.ReadAll(resp.Body)
406 c.Check(string(body), Equals, "")
407 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
408 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
412 resp, err := http.Get(
413 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
414 c.Check(err, Equals, nil)
415 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
416 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
420 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
422 defer closeListener()
425 client := http.Client{}
426 req, err := http.NewRequest("POST",
427 "http://"+listener.Addr().String()+"/",
428 strings.NewReader("qux"))
430 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
431 req.Header.Add("Content-Type", "application/octet-stream")
432 resp, err := client.Do(req)
433 c.Check(err, Equals, nil)
434 body, err := ioutil.ReadAll(resp.Body)
435 c.Check(err, Equals, nil)
436 c.Check(string(body), Matches,
437 fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
441 func (s *ServerRequiredSuite) TestStripHint(c *C) {
442 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
444 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
445 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
447 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
448 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
450 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
451 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
453 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
458 // Put one block, with 2 replicas
459 // With no prefix (expect the block locator, twice)
460 // With an existing prefix (expect the block locator, twice)
461 // With a valid but non-existing prefix (expect "\n")
462 // With an invalid prefix (expect error)
463 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
464 kc := runProxy(c, false)
465 defer closeListener()
467 // Put "index-data" blocks
468 data := []byte("index-data")
469 hash := fmt.Sprintf("%x", md5.Sum(data))
471 hash2, rep, err := kc.PutB(data)
472 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
473 c.Check(rep, Equals, 2)
474 c.Check(err, Equals, nil)
476 reader, blocklen, _, err := kc.Get(hash)
478 c.Check(blocklen, Equals, int64(10))
479 all, err := ioutil.ReadAll(reader)
481 c.Check(all, DeepEquals, data)
483 // Put some more blocks
484 _, _, err = kc.PutB([]byte("some-more-index-data"))
487 kc.Arvados.ApiToken = arvadostest.DataManagerToken
490 for _, spec := range []struct {
495 {"", true, true}, // with no prefix
496 {hash[:3], true, false}, // with matching prefix
497 {"abcdef", false, false}, // with no such prefix
499 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
500 c.Assert(err, Equals, nil)
501 indexResp, err := ioutil.ReadAll(indexReader)
502 c.Assert(err, Equals, nil)
503 locators := strings.Split(string(indexResp), "\n")
506 for _, locator := range locators {
510 c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
511 if locator[:32] == hash {
517 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
518 c.Check(gotOther > 0, Equals, spec.expectOther)
521 // GetIndex with invalid prefix
522 _, err = kc.GetIndex(TestProxyUUID, "xyz")
523 c.Assert((err != nil), Equals, true)
526 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
527 kc := runProxy(c, false)
528 defer closeListener()
529 hash, _, err := kc.PutB([]byte("shareddata"))
531 kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
532 rdr, _, _, err := kc.Get(hash)
534 data, err := ioutil.ReadAll(rdr)
536 c.Check(data, DeepEquals, []byte("shareddata"))
539 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
540 kc := runProxy(c, false)
541 defer closeListener()
544 hash, rep, err := kc.PutB([]byte("foo"))
546 c.Check(rep, Equals, 2)
548 for _, badToken := range []string{
550 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
552 kc.Arvados.ApiToken = badToken
554 // Ask and Get will fail only if the upstream
555 // keepstore server checks for valid signatures.
556 // Without knowing the blob signing key, there is no
557 // way for keepproxy to know whether a given token is
558 // permitted to read a block. So these tests fail:
560 _, _, err = kc.Ask(hash)
561 c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
562 c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
563 c.Check(err, ErrorMatches, ".*HTTP 403.*")
565 _, _, _, err = kc.Get(hash)
566 c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
567 c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
568 c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
571 _, _, err = kc.PutB([]byte("foo"))
572 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
576 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
577 kc := runProxy(c, false)
578 defer closeListener()
580 // Point keepproxy at a non-existent keepstore
581 locals := map[string]string{
582 TestProxyUUID: "http://localhost:12345",
584 router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
586 // Ask should result in temporary bad gateway error
587 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
588 _, _, err := kc.Ask(hash)
590 errNotFound, _ := err.(*keepclient.ErrNotFound)
591 c.Check(errNotFound.Temporary(), Equals, true)
592 c.Assert(err, ErrorMatches, ".*HTTP 502.*")
594 // Get should result in temporary bad gateway error
595 _, _, _, err = kc.Get(hash)
597 errNotFound, _ = err.(*keepclient.ErrNotFound)
598 c.Check(errNotFound.Temporary(), Equals, true)
599 c.Assert(err, ErrorMatches, ".*HTTP 502.*")
602 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
603 kc := runProxy(c, false)
604 defer closeListener()
606 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
607 for _, f := range []func() error{
609 _, _, err := kc.Ask(hash)
613 _, _, _, err := kc.Get(hash)
618 c.Assert(err, NotNil)
619 errNotFound, _ := err.(*keepclient.ErrNotFound)
620 c.Check(errNotFound.Temporary(), Equals, true)
621 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
625 func (s *ServerRequiredSuite) TestPing(c *C) {
626 kc := runProxy(c, false)
627 defer closeListener()
629 rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
631 req, err := http.NewRequest("GET",
632 "http://"+listener.Addr().String()+"/_health/ping",
635 req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
637 resp := httptest.NewRecorder()
638 rtr.ServeHTTP(resp, req)
639 c.Check(resp.Code, Equals, 200)
640 c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)