7 "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
8 "git.curoverse.com/arvados.git/sdk/go/arvadostest"
9 "git.curoverse.com/arvados.git/sdk/go/keepclient"
22 // Gocheck boilerplate
23 func Test(t *testing.T) {
27 // Gocheck boilerplate
28 var _ = Suite(&ServerRequiredSuite{})
30 // Tests that require the Keep server running
31 type ServerRequiredSuite struct{}
33 // Gocheck boilerplate
34 var _ = Suite(&NoKeepServerSuite{})
36 // Test with no keepserver to simulate errors
37 type NoKeepServerSuite struct{}
39 // Wait (up to 1 second) for keepproxy to listen on a port. This
40 // avoids a race condition where we hit a "connection refused" error
41 // because we start testing the proxy too soon.
42 func waitForListener() {
46 for i := 0; listener == nil && i < 1000; i += ms {
47 time.Sleep(ms * time.Millisecond)
50 log.Fatalf("Timed out waiting for listener to start")
54 func closeListener() {
60 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
61 arvadostest.StartAPI()
62 arvadostest.StartKeep(2, false)
65 func (s *ServerRequiredSuite) SetUpTest(c *C) {
66 arvadostest.ResetEnv()
69 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
70 arvadostest.StopKeep(2)
74 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
75 arvadostest.StartAPI()
78 func (s *NoKeepServerSuite) SetUpTest(c *C) {
79 arvadostest.ResetEnv()
82 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
86 func setupProxyService() {
88 client := &http.Client{Transport: &http.Transport{
89 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
93 if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
96 req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
98 reader, writer := io.Pipe()
104 data.Set("keep_service", `{
105 "service_host": "localhost",
106 "service_port": 29950,
107 "service_ssl_flag": false,
108 "service_type": "proxy"
111 writer.Write([]byte(data.Encode()))
115 var resp *http.Response
116 if resp, err = client.Do(req); err != nil {
119 if resp.StatusCode != 200 {
124 func runProxy(c *C, args []string, port int, bogusClientToken bool) *keepclient.KeepClient {
125 if bogusClientToken {
126 os.Setenv("ARVADOS_API_TOKEN", "bogus-token")
128 arv, err := arvadosclient.MakeArvadosClient()
129 c.Assert(err, Equals, nil)
130 kc := keepclient.KeepClient{
134 Client: &http.Client{},
136 locals := map[string]string{
137 "proxy": fmt.Sprintf("http://localhost:%v", port),
139 writableLocals := map[string]string{
140 "proxy": fmt.Sprintf("http://localhost:%v", port),
142 kc.SetServiceRoots(locals, writableLocals, nil)
143 c.Check(kc.Using_proxy, Equals, true)
144 c.Check(len(kc.LocalRoots()), Equals, 1)
145 for _, root := range kc.LocalRoots() {
146 c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port))
148 log.Print("keepclient created")
149 if bogusClientToken {
150 arvadostest.ResetEnv()
154 os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
162 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
163 log.Print("TestPutAndGet start")
165 os.Args = []string{"keepproxy", "-listen=:29950"}
168 time.Sleep(100 * time.Millisecond)
172 os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
173 arv, err := arvadosclient.MakeArvadosClient()
174 c.Assert(err, Equals, nil)
175 kc, err := keepclient.MakeKeepClient(&arv)
176 c.Assert(err, Equals, nil)
177 c.Check(kc.Arvados.External, Equals, true)
178 c.Check(kc.Using_proxy, Equals, true)
179 c.Check(len(kc.LocalRoots()), Equals, 1)
180 for _, root := range kc.LocalRoots() {
181 c.Check(root, Equals, "http://localhost:29950")
183 os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
186 defer closeListener()
188 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
192 _, _, err := kc.Ask(hash)
193 c.Check(err, Equals, keepclient.BlockNotFound)
194 log.Print("Finished Ask (expected BlockNotFound)")
198 reader, _, _, err := kc.Get(hash)
199 c.Check(reader, Equals, nil)
200 c.Check(err, Equals, keepclient.BlockNotFound)
201 log.Print("Finished Get (expected BlockNotFound)")
204 // Note in bug #5309 among other errors keepproxy would set
205 // Content-Length incorrectly on the 404 BlockNotFound response, this
206 // would result in a protocol violation that would prevent reuse of the
207 // connection, which would manifest by the next attempt to use the
208 // connection (in this case the PutB below) failing. So to test for
209 // that bug it's necessary to trigger an error response (such as
210 // BlockNotFound) and then do something else with the same httpClient
216 hash2, rep, err = kc.PutB([]byte("foo"))
217 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
218 c.Check(rep, Equals, 2)
219 c.Check(err, Equals, nil)
220 log.Print("Finished PutB (expected success)")
224 blocklen, _, err := kc.Ask(hash2)
225 c.Assert(err, Equals, nil)
226 c.Check(blocklen, Equals, int64(3))
227 log.Print("Finished Ask (expected success)")
231 reader, blocklen, _, err := kc.Get(hash2)
232 c.Assert(err, Equals, nil)
233 all, err := ioutil.ReadAll(reader)
234 c.Check(all, DeepEquals, []byte("foo"))
235 c.Check(blocklen, Equals, int64(3))
236 log.Print("Finished Get (expected success)")
242 hash2, rep, err = kc.PutB([]byte(""))
243 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
244 c.Check(rep, Equals, 2)
245 c.Check(err, Equals, nil)
246 log.Print("Finished PutB zero block")
250 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
251 c.Assert(err, Equals, nil)
252 all, err := ioutil.ReadAll(reader)
253 c.Check(all, DeepEquals, []byte(""))
254 c.Check(blocklen, Equals, int64(0))
255 log.Print("Finished Get zero block")
258 log.Print("TestPutAndGet done")
261 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
262 log.Print("TestPutAskGetForbidden start")
264 kc := runProxy(c, []string{"keepproxy"}, 29951, true)
266 defer closeListener()
268 hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
271 _, _, err := kc.Ask(hash)
272 errNotFound, _ := err.(keepclient.ErrNotFound)
273 c.Check(errNotFound, NotNil)
274 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
279 hash2, rep, err := kc.PutB([]byte("bar"))
280 c.Check(hash2, Equals, "")
281 c.Check(rep, Equals, 0)
282 c.Check(err, Equals, keepclient.InsufficientReplicasError)
287 blocklen, _, err := kc.Ask(hash)
288 errNotFound, _ := err.(keepclient.ErrNotFound)
289 c.Check(errNotFound, NotNil)
290 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
291 c.Check(blocklen, Equals, int64(0))
296 _, blocklen, _, err := kc.Get(hash)
297 errNotFound, _ := err.(keepclient.ErrNotFound)
298 c.Check(errNotFound, NotNil)
299 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
300 c.Check(blocklen, Equals, int64(0))
304 log.Print("TestPutAskGetForbidden done")
307 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
308 log.Print("TestGetDisabled start")
310 kc := runProxy(c, []string{"keepproxy", "-no-get"}, 29952, false)
312 defer closeListener()
314 hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
317 _, _, err := kc.Ask(hash)
318 errNotFound, _ := err.(keepclient.ErrNotFound)
319 c.Check(errNotFound, NotNil)
320 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
325 hash2, rep, err := kc.PutB([]byte("baz"))
326 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
327 c.Check(rep, Equals, 2)
328 c.Check(err, Equals, nil)
333 blocklen, _, err := kc.Ask(hash)
334 errNotFound, _ := err.(keepclient.ErrNotFound)
335 c.Check(errNotFound, NotNil)
336 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
337 c.Check(blocklen, Equals, int64(0))
342 _, blocklen, _, err := kc.Get(hash)
343 errNotFound, _ := err.(keepclient.ErrNotFound)
344 c.Check(errNotFound, NotNil)
345 c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
346 c.Check(blocklen, Equals, int64(0))
350 log.Print("TestGetDisabled done")
353 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
354 log.Print("TestPutDisabled start")
356 kc := runProxy(c, []string{"keepproxy", "-no-put"}, 29953, false)
358 defer closeListener()
361 hash2, rep, err := kc.PutB([]byte("quux"))
362 c.Check(hash2, Equals, "")
363 c.Check(rep, Equals, 0)
364 c.Check(err, Equals, keepclient.InsufficientReplicasError)
368 log.Print("TestPutDisabled done")
371 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
372 runProxy(c, []string{"keepproxy"}, 29954, false)
374 defer closeListener()
377 client := http.Client{}
378 req, err := http.NewRequest("OPTIONS",
379 fmt.Sprintf("http://localhost:29954/%x+3",
380 md5.Sum([]byte("foo"))),
382 req.Header.Add("Access-Control-Request-Method", "PUT")
383 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
384 resp, err := client.Do(req)
385 c.Check(err, Equals, nil)
386 c.Check(resp.StatusCode, Equals, 200)
387 body, err := ioutil.ReadAll(resp.Body)
388 c.Check(string(body), Equals, "")
389 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
390 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
394 resp, err := http.Get(
395 fmt.Sprintf("http://localhost:29954/%x+3",
396 md5.Sum([]byte("foo"))))
397 c.Check(err, Equals, nil)
398 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
399 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
403 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
404 runProxy(c, []string{"keepproxy"}, 29955, false)
406 defer closeListener()
409 client := http.Client{}
410 req, err := http.NewRequest("POST",
411 "http://localhost:29955/",
412 strings.NewReader("qux"))
413 req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
414 req.Header.Add("Content-Type", "application/octet-stream")
415 resp, err := client.Do(req)
416 c.Check(err, Equals, nil)
417 body, err := ioutil.ReadAll(resp.Body)
418 c.Check(err, Equals, nil)
419 c.Check(string(body), Matches,
420 fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
424 func (s *ServerRequiredSuite) TestStripHint(c *C) {
425 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
427 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
428 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
430 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
431 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
433 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
434 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
436 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
441 // Put one block, with 2 replicas
442 // With no prefix (expect the block locator, twice)
443 // With an existing prefix (expect the block locator, twice)
444 // With a valid but non-existing prefix (expect "\n")
445 // With an invalid prefix (expect error)
446 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
447 kc := runProxy(c, []string{"keepproxy"}, 28852, false)
449 defer closeListener()
451 // Put "index-data" blocks
452 data := []byte("index-data")
453 hash := fmt.Sprintf("%x", md5.Sum(data))
455 hash2, rep, err := kc.PutB(data)
456 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
457 c.Check(rep, Equals, 2)
458 c.Check(err, Equals, nil)
460 reader, blocklen, _, err := kc.Get(hash)
461 c.Assert(err, Equals, nil)
462 c.Check(blocklen, Equals, int64(10))
463 all, err := ioutil.ReadAll(reader)
464 c.Check(all, DeepEquals, data)
466 // Put some more blocks
467 _, rep, err = kc.PutB([]byte("some-more-index-data"))
468 c.Check(err, Equals, nil)
471 for _, spec := range []struct {
476 {"", true, true}, // with no prefix
477 {hash[:3], true, false}, // with matching prefix
478 {"abcdef", false, false}, // with no such prefix
480 indexReader, err := kc.GetIndex("proxy", spec.prefix)
481 c.Assert(err, Equals, nil)
482 indexResp, err := ioutil.ReadAll(indexReader)
483 c.Assert(err, Equals, nil)
484 locators := strings.Split(string(indexResp), "\n")
487 for _, locator := range locators {
491 c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
492 if locator[:32] == hash {
498 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
499 c.Check(gotOther > 0, Equals, spec.expectOther)
502 // GetIndex with invalid prefix
503 _, err = kc.GetIndex("proxy", "xyz")
504 c.Assert((err != nil), Equals, true)
507 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
508 kc := runProxy(c, []string{"keepproxy"}, 28852, false)
510 defer closeListener()
513 hash, rep, err := kc.PutB([]byte("foo"))
514 c.Check(err, Equals, nil)
515 c.Check(rep, Equals, 2)
517 for _, token := range []string{
519 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
521 // Change token to given bad token
522 kc.Arvados.ApiToken = token
524 // Ask should result in error
525 _, _, err = kc.Ask(hash)
527 errNotFound, _ := err.(keepclient.ErrNotFound)
528 c.Check(errNotFound.Temporary(), Equals, false)
529 c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true)
531 // Get should result in error
532 _, _, _, err = kc.Get(hash)
534 errNotFound, _ = err.(keepclient.ErrNotFound)
535 c.Check(errNotFound.Temporary(), Equals, false)
536 c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true)
540 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
541 arv, err := arvadosclient.MakeArvadosClient()
542 c.Assert(err, Equals, nil)
544 // keepclient with no such keep server
545 kc := keepclient.New(&arv)
546 locals := map[string]string{
547 "proxy": "http://localhost:12345",
549 kc.SetServiceRoots(locals, nil, nil)
551 // Ask should result in temporary connection refused error
552 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
553 _, _, err = kc.Ask(hash)
555 errNotFound, _ := err.(*keepclient.ErrNotFound)
556 c.Check(errNotFound.Temporary(), Equals, true)
557 c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
559 // Get should result in temporary connection refused error
560 _, _, _, err = kc.Get(hash)
562 errNotFound, _ = err.(*keepclient.ErrNotFound)
563 c.Check(errNotFound.Temporary(), Equals, true)
564 c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
567 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
568 kc := runProxy(c, []string{"keepproxy"}, 29999, false)
570 defer closeListener()
572 // Ask should result in temporary connection refused error
573 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
574 _, _, err := kc.Ask(hash)
576 errNotFound, _ := err.(*keepclient.ErrNotFound)
577 c.Check(errNotFound.Temporary(), Equals, true)
578 c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true)
580 // Get should result in temporary connection refused error
581 _, _, _, err = kc.Get(hash)
583 errNotFound, _ = err.(*keepclient.ErrNotFound)
584 c.Check(errNotFound.Temporary(), Equals, true)
585 c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true)