1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
21 "git.arvados.org/arvados.git/lib/config"
22 "git.arvados.org/arvados.git/sdk/go/arvados"
23 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
24 "git.arvados.org/arvados.git/sdk/go/arvadostest"
25 "git.arvados.org/arvados.git/sdk/go/ctxlog"
26 "git.arvados.org/arvados.git/sdk/go/keepclient"
27 log "github.com/sirupsen/logrus"
32 // Gocheck boilerplate
33 func Test(t *testing.T) {
37 // Gocheck boilerplate
38 var _ = Suite(&ServerRequiredSuite{})
40 // Tests that require the Keep server running
41 type ServerRequiredSuite struct{}
43 // Gocheck boilerplate
44 var _ = Suite(&NoKeepServerSuite{})
46 // Test with no keepserver to simulate errors
47 type NoKeepServerSuite struct{}
49 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
51 // Wait (up to 1 second) for keepproxy to listen on a port. This
52 // avoids a race condition where we hit a "connection refused" error
53 // because we start testing the proxy too soon.
54 func waitForListener() {
58 for i := 0; listener == nil && i < 10000; i += ms {
59 time.Sleep(ms * time.Millisecond)
62 panic("Timed out waiting for listener to start")
66 func closeListener() {
72 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
73 arvadostest.StartAPI()
74 arvadostest.StartKeep(2, false)
77 func (s *ServerRequiredSuite) SetUpTest(c *C) {
78 arvadostest.ResetEnv()
81 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
82 arvadostest.StopKeep(2)
86 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
87 arvadostest.StartAPI()
88 // We need API to have some keep services listed, but the
89 // services themselves should be unresponsive.
90 arvadostest.StartKeep(2, false)
91 arvadostest.StopKeep(2)
94 func (s *NoKeepServerSuite) SetUpTest(c *C) {
95 arvadostest.ResetEnv()
98 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
102 func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
103 cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
104 c.Assert(err, Equals, nil)
105 cluster, err := cfg.GetCluster("")
106 c.Assert(err, Equals, nil)
108 cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
112 run(log.New(), cluster)
113 defer closeListener()
117 client := arvados.NewClientFromEnv()
118 arv, err := arvadosclient.New(client)
119 c.Assert(err, Equals, nil)
120 if bogusClientToken {
121 arv.ApiToken = "bogus-token"
123 kc := keepclient.New(arv)
124 sr := map[string]string{
125 TestProxyUUID: "http://" + listener.Addr().String(),
127 kc.SetServiceRoots(sr, sr, sr)
128 kc.Arvados.External = true
133 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
135 defer closeListener()
137 req, err := http.NewRequest("POST",
138 "http://"+listener.Addr().String()+"/",
139 strings.NewReader("TestViaHeader"))
140 c.Assert(err, Equals, nil)
141 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
142 resp, err := (&http.Client{}).Do(req)
143 c.Assert(err, Equals, nil)
144 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
145 c.Assert(resp.StatusCode, Equals, http.StatusOK)
146 locator, err := ioutil.ReadAll(resp.Body)
147 c.Assert(err, Equals, nil)
150 req, err = http.NewRequest("GET",
151 "http://"+listener.Addr().String()+"/"+string(locator),
153 c.Assert(err, Equals, nil)
154 resp, err = (&http.Client{}).Do(req)
155 c.Assert(err, Equals, nil)
156 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
160 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
161 kc := runProxy(c, false)
162 defer closeListener()
164 sr := map[string]string{
165 TestProxyUUID: "http://" + listener.Addr().String(),
167 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
169 content := []byte("TestLoopDetection")
170 _, _, err := kc.PutB(content)
171 c.Check(err, ErrorMatches, `.*loop detected.*`)
173 hash := fmt.Sprintf("%x", md5.Sum(content))
174 _, _, _, err = kc.Get(hash)
175 c.Check(err, ErrorMatches, `.*loop detected.*`)
178 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
179 kc := runProxy(c, false)
180 defer closeListener()
182 // Set up fake keepstore to record request headers
184 ts := httptest.NewServer(http.HandlerFunc(
185 func(w http.ResponseWriter, r *http.Request) {
187 http.Error(w, "Error", http.StatusInternalServerError)
191 // Point keepproxy router's keepclient to the fake keepstore
192 sr := map[string]string{
193 TestProxyUUID: ts.URL,
195 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
197 // Set up client to ask for storage classes to keepproxy
198 kc.StorageClasses = []string{"secure"}
199 content := []byte("Very important data")
200 _, _, err := kc.PutB(content)
202 c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
205 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
206 kc := runProxy(c, false)
207 defer closeListener()
209 content := []byte("TestDesiredReplicas")
210 hash := fmt.Sprintf("%x", md5.Sum(content))
212 for _, kc.Want_replicas = range []int{0, 1, 2} {
213 locator, rep, err := kc.PutB(content)
214 c.Check(err, Equals, nil)
215 c.Check(rep, Equals, kc.Want_replicas)
217 c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
222 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
223 kc := runProxy(c, false)
224 defer closeListener()
226 content := []byte("TestPutWrongContentLength")
227 hash := fmt.Sprintf("%x", md5.Sum(content))
229 // If we use http.Client to send these requests to the network
230 // server we just started, the Go http library automatically
231 // fixes the invalid Content-Length header. In order to test
232 // our server behavior, we have to call the handler directly
233 // using an httptest.ResponseRecorder.
234 rtr := MakeRESTRouter(kc, 10*time.Second, "")
236 type testcase struct {
241 for _, t := range []testcase{
242 {"1", http.StatusBadRequest},
243 {"", http.StatusLengthRequired},
244 {"-1", http.StatusLengthRequired},
245 {"abcdef", http.StatusLengthRequired},
247 req, err := http.NewRequest("PUT",
248 fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
249 bytes.NewReader(content))
251 req.Header.Set("Content-Length", t.sendLength)
252 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
253 req.Header.Set("Content-Type", "application/octet-stream")
255 resp := httptest.NewRecorder()
256 rtr.ServeHTTP(resp, req)
257 c.Check(resp.Code, Equals, t.expectStatus)
261 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
262 kc := runProxy(c, false)
263 defer closeListener()
264 router.(*proxyHandler).timeout = time.Nanosecond
266 buf := make([]byte, 1<<20)
268 var wg sync.WaitGroup
269 for i := 0; i < 128; i++ {
276 done := make(chan bool)
283 case <-time.After(10 * time.Second):
288 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
289 kc := runProxy(c, false)
290 defer closeListener()
292 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
296 _, _, err := kc.Ask(hash)
297 c.Check(err, Equals, keepclient.BlockNotFound)
298 c.Log("Finished Ask (expected BlockNotFound)")
302 reader, _, _, err := kc.Get(hash)
303 c.Check(reader, Equals, nil)
304 c.Check(err, Equals, keepclient.BlockNotFound)
305 c.Log("Finished Get (expected BlockNotFound)")
308 // Note in bug #5309 among other errors keepproxy would set
309 // Content-Length incorrectly on the 404 BlockNotFound response, this
310 // would result in a protocol violation that would prevent reuse of the
311 // connection, which would manifest by the next attempt to use the
312 // connection (in this case the PutB below) failing. So to test for
313 // that bug it's necessary to trigger an error response (such as
314 // BlockNotFound) and then do something else with the same httpClient
320 hash2, rep, err = kc.PutB([]byte("foo"))
321 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
322 c.Check(rep, Equals, 2)
323 c.Check(err, Equals, nil)
324 c.Log("Finished PutB (expected success)")
328 blocklen, _, err := kc.Ask(hash2)
329 c.Assert(err, Equals, nil)
330 c.Check(blocklen, Equals, int64(3))
331 c.Log("Finished Ask (expected success)")
335 reader, blocklen, _, err := kc.Get(hash2)
336 c.Assert(err, Equals, nil)
337 all, err := ioutil.ReadAll(reader)
339 c.Check(all, DeepEquals, []byte("foo"))
340 c.Check(blocklen, Equals, int64(3))
341 c.Log("Finished Get (expected success)")
347 hash2, rep, err = kc.PutB([]byte(""))
348 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
349 c.Check(rep, Equals, 2)
350 c.Check(err, Equals, nil)
351 c.Log("Finished PutB zero block")
355 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
356 c.Assert(err, Equals, nil)
357 all, err := ioutil.ReadAll(reader)
359 c.Check(all, DeepEquals, []byte(""))
360 c.Check(blocklen, Equals, int64(0))
361 c.Log("Finished Get zero block")
365 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
366 kc := runProxy(c, true)
367 defer closeListener()
369 hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
371 _, _, err := kc.Ask(hash)
372 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
374 hash2, rep, err := kc.PutB([]byte("bar"))
375 c.Check(hash2, Equals, "")
376 c.Check(rep, Equals, 0)
377 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
379 blocklen, _, err := kc.Ask(hash)
380 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
381 c.Check(err, ErrorMatches, ".*not found.*")
382 c.Check(blocklen, Equals, int64(0))
384 _, blocklen, _, err = kc.Get(hash)
385 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
386 c.Check(err, ErrorMatches, ".*not found.*")
387 c.Check(blocklen, Equals, int64(0))
391 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
393 defer closeListener()
396 client := http.Client{}
397 req, err := http.NewRequest("OPTIONS",
398 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
401 req.Header.Add("Access-Control-Request-Method", "PUT")
402 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
403 resp, err := client.Do(req)
404 c.Check(err, Equals, nil)
405 c.Check(resp.StatusCode, Equals, 200)
406 body, err := ioutil.ReadAll(resp.Body)
408 c.Check(string(body), Equals, "")
409 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
410 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
414 resp, err := http.Get(
415 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
416 c.Check(err, Equals, nil)
417 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
418 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
422 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
424 defer closeListener()
427 client := http.Client{}
428 req, err := http.NewRequest("POST",
429 "http://"+listener.Addr().String()+"/",
430 strings.NewReader("qux"))
432 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
433 req.Header.Add("Content-Type", "application/octet-stream")
434 resp, err := client.Do(req)
435 c.Check(err, Equals, nil)
436 body, err := ioutil.ReadAll(resp.Body)
437 c.Check(err, Equals, nil)
438 c.Check(string(body), Matches,
439 fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
443 func (s *ServerRequiredSuite) TestStripHint(c *C) {
444 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
446 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
447 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
449 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
450 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
452 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
453 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
455 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
460 // Put one block, with 2 replicas
461 // With no prefix (expect the block locator, twice)
462 // With an existing prefix (expect the block locator, twice)
463 // With a valid but non-existing prefix (expect "\n")
464 // With an invalid prefix (expect error)
465 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
466 kc := runProxy(c, false)
467 defer closeListener()
469 // Put "index-data" blocks
470 data := []byte("index-data")
471 hash := fmt.Sprintf("%x", md5.Sum(data))
473 hash2, rep, err := kc.PutB(data)
474 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
475 c.Check(rep, Equals, 2)
476 c.Check(err, Equals, nil)
478 reader, blocklen, _, err := kc.Get(hash)
480 c.Check(blocklen, Equals, int64(10))
481 all, err := ioutil.ReadAll(reader)
483 c.Check(all, DeepEquals, data)
485 // Put some more blocks
486 _, _, err = kc.PutB([]byte("some-more-index-data"))
489 kc.Arvados.ApiToken = arvadostest.SystemRootToken
492 for _, spec := range []struct {
497 {"", true, true}, // with no prefix
498 {hash[:3], true, false}, // with matching prefix
499 {"abcdef", false, false}, // with no such prefix
501 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
502 c.Assert(err, Equals, nil)
503 indexResp, err := ioutil.ReadAll(indexReader)
504 c.Assert(err, Equals, nil)
505 locators := strings.Split(string(indexResp), "\n")
508 for _, locator := range locators {
512 c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
513 if locator[:32] == hash {
519 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
520 c.Check(gotOther > 0, Equals, spec.expectOther)
523 // GetIndex with invalid prefix
524 _, err = kc.GetIndex(TestProxyUUID, "xyz")
525 c.Assert((err != nil), Equals, true)
528 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
529 kc := runProxy(c, false)
530 defer closeListener()
531 hash, _, err := kc.PutB([]byte("shareddata"))
533 kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
534 rdr, _, _, err := kc.Get(hash)
536 data, err := ioutil.ReadAll(rdr)
538 c.Check(data, DeepEquals, []byte("shareddata"))
541 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
542 kc := runProxy(c, false)
543 defer closeListener()
546 hash, rep, err := kc.PutB([]byte("foo"))
548 c.Check(rep, Equals, 2)
550 for _, badToken := range []string{
552 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
554 kc.Arvados.ApiToken = badToken
556 // Ask and Get will fail only if the upstream
557 // keepstore server checks for valid signatures.
558 // Without knowing the blob signing key, there is no
559 // way for keepproxy to know whether a given token is
560 // permitted to read a block. So these tests fail:
562 _, _, err = kc.Ask(hash)
563 c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
564 c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
565 c.Check(err, ErrorMatches, ".*HTTP 403.*")
567 _, _, _, err = kc.Get(hash)
568 c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
569 c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
570 c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
573 _, _, err = kc.PutB([]byte("foo"))
574 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
578 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
579 kc := runProxy(c, false)
580 defer closeListener()
582 // Point keepproxy at a non-existent keepstore
583 locals := map[string]string{
584 TestProxyUUID: "http://localhost:12345",
586 router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
588 // Ask should result in temporary bad gateway error
589 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
590 _, _, err := kc.Ask(hash)
592 errNotFound, _ := err.(*keepclient.ErrNotFound)
593 c.Check(errNotFound.Temporary(), Equals, true)
594 c.Assert(err, ErrorMatches, ".*HTTP 502.*")
596 // Get should result in temporary bad gateway error
597 _, _, _, err = kc.Get(hash)
599 errNotFound, _ = err.(*keepclient.ErrNotFound)
600 c.Check(errNotFound.Temporary(), Equals, true)
601 c.Assert(err, ErrorMatches, ".*HTTP 502.*")
604 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
605 kc := runProxy(c, false)
606 defer closeListener()
608 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
609 for _, f := range []func() error{
611 _, _, err := kc.Ask(hash)
615 _, _, _, err := kc.Get(hash)
620 c.Assert(err, NotNil)
621 errNotFound, _ := err.(*keepclient.ErrNotFound)
622 c.Check(errNotFound.Temporary(), Equals, true)
623 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
627 func (s *ServerRequiredSuite) TestPing(c *C) {
628 kc := runProxy(c, false)
629 defer closeListener()
631 rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
633 req, err := http.NewRequest("GET",
634 "http://"+listener.Addr().String()+"/_health/ping",
637 req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
639 resp := httptest.NewRecorder()
640 rtr.ServeHTTP(resp, req)
641 c.Check(resp.Code, Equals, 200)
642 c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)