17 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
18 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
19 "git.curoverse.com/arvados.git/sdk/go/keepclient"
24 // Gocheck boilerplate
25 func Test(t *testing.T) {
29 // Gocheck boilerplate
30 var _ = Suite(&ServerRequiredSuite{})
32 // Tests that require the Keep server running
33 type ServerRequiredSuite struct{}
35 // Gocheck boilerplate
36 var _ = Suite(&NoKeepServerSuite{})
38 // Test with no keepserver to simulate errors
39 type NoKeepServerSuite struct{}
41 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
43 // Wait (up to 1 second) for keepproxy to listen on a port. This
44 // avoids a race condition where we hit a "connection refused" error
45 // because we start testing the proxy too soon.
46 func waitForListener() {
50 for i := 0; listener == nil && i < 10000; i += ms {
51 time.Sleep(ms * time.Millisecond)
54 log.Fatalf("Timed out waiting for listener to start")
58 func closeListener() {
64 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
65 arvadostest.StartAPI()
66 arvadostest.StartKeep(2, false)
69 func (s *ServerRequiredSuite) SetUpTest(c *C) {
70 arvadostest.ResetEnv()
73 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
74 arvadostest.StopKeep(2)
78 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
79 arvadostest.StartAPI()
80 // We need API to have some keep services listed, but the
81 // services themselves should be unresponsive.
82 arvadostest.StartKeep(2, false)
83 arvadostest.StopKeep(2)
86 func (s *NoKeepServerSuite) SetUpTest(c *C) {
87 arvadostest.ResetEnv()
90 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
94 func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
95 args = append([]string{"keepproxy"}, args...)
96 os.Args = append(args, "-listen=:0")
101 arv, err := arvadosclient.MakeArvadosClient()
102 c.Assert(err, Equals, nil)
103 if bogusClientToken {
104 arv.ApiToken = "bogus-token"
106 kc := keepclient.New(arv)
107 sr := map[string]string{
108 TestProxyUUID: "http://" + listener.Addr().String(),
110 kc.SetServiceRoots(sr, sr, sr)
111 kc.Arvados.External = true
116 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
117 kc := runProxy(c, nil, false)
118 defer closeListener()
120 sr := map[string]string{
121 TestProxyUUID: "http://" + listener.Addr().String(),
123 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
125 content := []byte("TestLoopDetection")
126 _, _, err := kc.PutB(content)
127 c.Check(err, ErrorMatches, `.*loop detected.*`)
129 hash := fmt.Sprintf("%x", md5.Sum(content))
130 _, _, _, err = kc.Get(hash)
131 c.Check(err, ErrorMatches, `.*loop detected.*`)
134 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
135 kc := runProxy(c, nil, false)
136 defer closeListener()
138 content := []byte("TestDesiredReplicas")
139 hash := fmt.Sprintf("%x", md5.Sum(content))
141 for _, kc.Want_replicas = range []int{0, 1, 2} {
142 locator, rep, err := kc.PutB(content)
143 c.Check(err, Equals, nil)
144 c.Check(rep, Equals, kc.Want_replicas)
146 c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
151 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
152 kc := runProxy(c, nil, false)
153 defer closeListener()
155 content := []byte("TestPutWrongContentLength")
156 hash := fmt.Sprintf("%x", md5.Sum(content))
158 // If we use http.Client to send these requests to the network
159 // server we just started, the Go http library automatically
160 // fixes the invalid Content-Length header. In order to test
161 // our server behavior, we have to call the handler directly
162 // using an httptest.ResponseRecorder.
163 rtr := MakeRESTRouter(true, true, kc)
165 type testcase struct {
170 for _, t := range []testcase{
171 {"1", http.StatusBadRequest},
172 {"", http.StatusLengthRequired},
173 {"-1", http.StatusLengthRequired},
174 {"abcdef", http.StatusLengthRequired},
176 req, err := http.NewRequest("PUT",
177 fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
178 bytes.NewReader(content))
180 req.Header.Set("Content-Length", t.sendLength)
181 req.Header.Set("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
182 req.Header.Set("Content-Type", "application/octet-stream")
184 resp := httptest.NewRecorder()
185 rtr.ServeHTTP(resp, req)
186 c.Check(resp.Code, Equals, t.expectStatus)
190 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
191 kc := runProxy(c, nil, false)
192 defer closeListener()
194 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
198 _, _, err := kc.Ask(hash)
199 c.Check(err, Equals, keepclient.BlockNotFound)
200 log.Print("Finished Ask (expected BlockNotFound)")
204 reader, _, _, err := kc.Get(hash)
205 c.Check(reader, Equals, nil)
206 c.Check(err, Equals, keepclient.BlockNotFound)
207 log.Print("Finished Get (expected BlockNotFound)")
210 // Note in bug #5309 among other errors keepproxy would set
211 // Content-Length incorrectly on the 404 BlockNotFound response, this
212 // would result in a protocol violation that would prevent reuse of the
213 // connection, which would manifest by the next attempt to use the
214 // connection (in this case the PutB below) failing. So to test for
215 // that bug it's necessary to trigger an error response (such as
216 // BlockNotFound) and then do something else with the same httpClient
222 hash2, rep, err = kc.PutB([]byte("foo"))
223 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
224 c.Check(rep, Equals, 2)
225 c.Check(err, Equals, nil)
226 log.Print("Finished PutB (expected success)")
230 blocklen, _, err := kc.Ask(hash2)
231 c.Assert(err, Equals, nil)
232 c.Check(blocklen, Equals, int64(3))
233 log.Print("Finished Ask (expected success)")
237 reader, blocklen, _, err := kc.Get(hash2)
238 c.Assert(err, Equals, nil)
239 all, err := ioutil.ReadAll(reader)
240 c.Check(all, DeepEquals, []byte("foo"))
241 c.Check(blocklen, Equals, int64(3))
242 log.Print("Finished Get (expected success)")
248 hash2, rep, err = kc.PutB([]byte(""))
249 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
250 c.Check(rep, Equals, 2)
251 c.Check(err, Equals, nil)
252 log.Print("Finished PutB zero block")
256 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
257 c.Assert(err, Equals, nil)
258 all, err := ioutil.ReadAll(reader)
259 c.Check(all, DeepEquals, []byte(""))
260 c.Check(blocklen, Equals, int64(0))
261 log.Print("Finished Get zero block")
265 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
266 kc := runProxy(c, nil, true)
267 defer closeListener()
269 hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
272 _, _, err := kc.Ask(hash)
273 errNotFound, _ := err.(keepclient.ErrNotFound)
274 c.Check(errNotFound, NotNil)
275 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
280 hash2, rep, err := kc.PutB([]byte("bar"))
281 c.Check(hash2, Equals, "")
282 c.Check(rep, Equals, 0)
283 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
288 blocklen, _, err := kc.Ask(hash)
289 errNotFound, _ := err.(keepclient.ErrNotFound)
290 c.Check(errNotFound, NotNil)
291 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
292 c.Check(blocklen, Equals, int64(0))
297 _, blocklen, _, err := kc.Get(hash)
298 errNotFound, _ := err.(keepclient.ErrNotFound)
299 c.Check(errNotFound, NotNil)
300 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
301 c.Check(blocklen, Equals, int64(0))
306 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
307 kc := runProxy(c, []string{"-no-get"}, false)
308 defer closeListener()
310 hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
313 _, _, err := kc.Ask(hash)
314 errNotFound, _ := err.(keepclient.ErrNotFound)
315 c.Check(errNotFound, NotNil)
316 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
321 hash2, rep, err := kc.PutB([]byte("baz"))
322 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
323 c.Check(rep, Equals, 2)
324 c.Check(err, Equals, nil)
329 blocklen, _, err := kc.Ask(hash)
330 errNotFound, _ := err.(keepclient.ErrNotFound)
331 c.Check(errNotFound, NotNil)
332 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
333 c.Check(blocklen, Equals, int64(0))
338 _, blocklen, _, err := kc.Get(hash)
339 errNotFound, _ := err.(keepclient.ErrNotFound)
340 c.Check(errNotFound, NotNil)
341 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
342 c.Check(blocklen, Equals, int64(0))
347 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
348 kc := runProxy(c, []string{"-no-put"}, false)
349 defer closeListener()
351 hash2, rep, err := kc.PutB([]byte("quux"))
352 c.Check(hash2, Equals, "")
353 c.Check(rep, Equals, 0)
354 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
357 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
358 runProxy(c, nil, false)
359 defer closeListener()
362 client := http.Client{}
363 req, err := http.NewRequest("OPTIONS",
364 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
366 req.Header.Add("Access-Control-Request-Method", "PUT")
367 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
368 resp, err := client.Do(req)
369 c.Check(err, Equals, nil)
370 c.Check(resp.StatusCode, Equals, 200)
371 body, err := ioutil.ReadAll(resp.Body)
372 c.Check(string(body), Equals, "")
373 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
374 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
378 resp, err := http.Get(
379 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
380 c.Check(err, Equals, nil)
381 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
382 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
386 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
387 runProxy(c, nil, false)
388 defer closeListener()
391 client := http.Client{}
392 req, err := http.NewRequest("POST",
393 "http://"+listener.Addr().String()+"/",
394 strings.NewReader("qux"))
395 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
396 req.Header.Add("Content-Type", "application/octet-stream")
397 resp, err := client.Do(req)
398 c.Check(err, Equals, nil)
399 body, err := ioutil.ReadAll(resp.Body)
400 c.Check(err, Equals, nil)
401 c.Check(string(body), Matches,
402 fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
406 func (s *ServerRequiredSuite) TestStripHint(c *C) {
407 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
409 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
410 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
412 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
413 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
415 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
416 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
418 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
423 // Put one block, with 2 replicas
424 // With no prefix (expect the block locator, twice)
425 // With an existing prefix (expect the block locator, twice)
426 // With a valid but non-existing prefix (expect "\n")
427 // With an invalid prefix (expect error)
428 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
429 kc := runProxy(c, nil, false)
430 defer closeListener()
432 // Put "index-data" blocks
433 data := []byte("index-data")
434 hash := fmt.Sprintf("%x", md5.Sum(data))
436 hash2, rep, err := kc.PutB(data)
437 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
438 c.Check(rep, Equals, 2)
439 c.Check(err, Equals, nil)
441 reader, blocklen, _, err := kc.Get(hash)
442 c.Assert(err, Equals, nil)
443 c.Check(blocklen, Equals, int64(10))
444 all, err := ioutil.ReadAll(reader)
445 c.Check(all, DeepEquals, data)
447 // Put some more blocks
448 _, rep, err = kc.PutB([]byte("some-more-index-data"))
449 c.Check(err, Equals, nil)
451 kc.Arvados.ApiToken = arvadostest.DataManagerToken
454 for _, spec := range []struct {
459 {"", true, true}, // with no prefix
460 {hash[:3], true, false}, // with matching prefix
461 {"abcdef", false, false}, // with no such prefix
463 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
464 c.Assert(err, Equals, nil)
465 indexResp, err := ioutil.ReadAll(indexReader)
466 c.Assert(err, Equals, nil)
467 locators := strings.Split(string(indexResp), "\n")
470 for _, locator := range locators {
474 c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
475 if locator[:32] == hash {
481 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
482 c.Check(gotOther > 0, Equals, spec.expectOther)
485 // GetIndex with invalid prefix
486 _, err = kc.GetIndex(TestProxyUUID, "xyz")
487 c.Assert((err != nil), Equals, true)
490 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
491 kc := runProxy(c, nil, false)
492 defer closeListener()
495 hash, rep, err := kc.PutB([]byte("foo"))
496 c.Check(err, Equals, nil)
497 c.Check(rep, Equals, 2)
499 for _, token := range []string{
501 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
503 // Change token to given bad token
504 kc.Arvados.ApiToken = token
506 // Ask should result in error
507 _, _, err = kc.Ask(hash)
509 errNotFound, _ := err.(keepclient.ErrNotFound)
510 c.Check(errNotFound.Temporary(), Equals, false)
511 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
513 // Get should result in error
514 _, _, _, err = kc.Get(hash)
516 errNotFound, _ = err.(keepclient.ErrNotFound)
517 c.Check(errNotFound.Temporary(), Equals, false)
518 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
522 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
523 arv, err := arvadosclient.MakeArvadosClient()
524 c.Assert(err, Equals, nil)
526 // keepclient with no such keep server
527 kc := keepclient.New(arv)
528 locals := map[string]string{
529 TestProxyUUID: "http://localhost:12345",
531 kc.SetServiceRoots(locals, nil, nil)
533 // Ask should result in temporary connection refused error
534 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
535 _, _, err = kc.Ask(hash)
537 errNotFound, _ := err.(*keepclient.ErrNotFound)
538 c.Check(errNotFound.Temporary(), Equals, true)
539 c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
541 // Get should result in temporary connection refused error
542 _, _, _, err = kc.Get(hash)
544 errNotFound, _ = err.(*keepclient.ErrNotFound)
545 c.Check(errNotFound.Temporary(), Equals, true)
546 c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
549 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
550 kc := runProxy(c, nil, false)
551 defer closeListener()
553 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
554 for _, f := range []func() error{
556 _, _, err := kc.Ask(hash)
560 _, _, _, err := kc.Get(hash)
565 c.Assert(err, NotNil)
566 errNotFound, _ := err.(*keepclient.ErrNotFound)
567 c.Check(errNotFound.Temporary(), Equals, true)
568 c.Check(err, ErrorMatches, `.*HTTP 502.*`)