1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
20 "git.arvados.org/arvados.git/lib/config"
21 "git.arvados.org/arvados.git/sdk/go/arvados"
22 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
23 "git.arvados.org/arvados.git/sdk/go/arvadostest"
24 "git.arvados.org/arvados.git/sdk/go/ctxlog"
25 "git.arvados.org/arvados.git/sdk/go/keepclient"
26 log "github.com/sirupsen/logrus"
32 // Gocheck boilerplate
33 func Test(t *testing.T) {
37 // Gocheck boilerplate
38 var _ = Suite(&ServerRequiredSuite{})
40 // Tests that require the Keep server running
41 type ServerRequiredSuite struct{}
43 // Gocheck boilerplate
44 var _ = Suite(&ServerRequiredConfigYmlSuite{})
46 // Tests that require the Keep servers running as defined in config.yml
47 type ServerRequiredConfigYmlSuite struct{}
49 // Gocheck boilerplate
50 var _ = Suite(&NoKeepServerSuite{})
52 // Test with no keepserver to simulate errors
53 type NoKeepServerSuite struct{}
55 var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
57 // Wait (up to 1 second) for keepproxy to listen on a port. This
58 // avoids a race condition where we hit a "connection refused" error
59 // because we start testing the proxy too soon.
60 func waitForListener() {
64 for i := 0; listener == nil && i < 10000; i += ms {
65 time.Sleep(ms * time.Millisecond)
68 panic("Timed out waiting for listener to start")
72 func closeListener() {
78 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
79 arvadostest.StartAPI()
80 arvadostest.StartKeep(2, false)
83 func (s *ServerRequiredSuite) SetUpTest(c *C) {
84 arvadostest.ResetEnv()
87 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
88 arvadostest.StopKeep(2)
92 func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
93 arvadostest.StartAPI()
94 // config.yml defines 4 keepstores
95 arvadostest.StartKeep(4, false)
98 func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
99 arvadostest.ResetEnv()
102 func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
103 arvadostest.StopKeep(4)
104 arvadostest.StopAPI()
107 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
108 arvadostest.StartAPI()
109 // We need API to have some keep services listed, but the
110 // services themselves should be unresponsive.
111 arvadostest.StartKeep(2, false)
112 arvadostest.StopKeep(2)
115 func (s *NoKeepServerSuite) SetUpTest(c *C) {
116 arvadostest.ResetEnv()
119 func (s *NoKeepServerSuite) TearDownSuite(c *C) {
120 arvadostest.StopAPI()
123 func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) {
124 cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
125 c.Assert(err, Equals, nil)
126 cluster, err := cfg.GetCluster("")
127 c.Assert(err, Equals, nil)
129 if !loadKeepstoresFromConfig {
130 // Do not load Keepstore InternalURLs from the config file
131 cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
134 cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: ":0"}: {}}
137 cluster.Collections.KeepproxyPermission = *kp
141 logbuf := &bytes.Buffer{}
146 defer closeListener()
150 client := arvados.NewClientFromEnv()
151 arv, err := arvadosclient.New(client)
152 c.Assert(err, Equals, nil)
153 if bogusClientToken {
154 arv.ApiToken = "bogus-token"
156 kc := keepclient.New(arv)
157 sr := map[string]string{
158 TestProxyUUID: "http://" + listener.Addr().String(),
160 kc.SetServiceRoots(sr, sr, sr)
161 kc.Arvados.External = true
166 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
167 runProxy(c, false, false, nil)
168 defer closeListener()
170 req, err := http.NewRequest("POST",
171 "http://"+listener.Addr().String()+"/",
172 strings.NewReader("TestViaHeader"))
173 c.Assert(err, Equals, nil)
174 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
175 resp, err := (&http.Client{}).Do(req)
176 c.Assert(err, Equals, nil)
177 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
178 c.Assert(resp.StatusCode, Equals, http.StatusOK)
179 locator, err := ioutil.ReadAll(resp.Body)
180 c.Assert(err, Equals, nil)
183 req, err = http.NewRequest("GET",
184 "http://"+listener.Addr().String()+"/"+string(locator),
186 c.Assert(err, Equals, nil)
187 resp, err = (&http.Client{}).Do(req)
188 c.Assert(err, Equals, nil)
189 c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
193 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
194 kc, _ := runProxy(c, false, false, nil)
195 defer closeListener()
197 sr := map[string]string{
198 TestProxyUUID: "http://" + listener.Addr().String(),
200 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
202 content := []byte("TestLoopDetection")
203 _, _, err := kc.PutB(content)
204 c.Check(err, ErrorMatches, `.*loop detected.*`)
206 hash := fmt.Sprintf("%x", md5.Sum(content))
207 _, _, _, err = kc.Get(hash)
208 c.Check(err, ErrorMatches, `.*loop detected.*`)
211 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
212 kc, _ := runProxy(c, false, false, nil)
213 defer closeListener()
215 // Set up fake keepstore to record request headers
217 ts := httptest.NewServer(http.HandlerFunc(
218 func(w http.ResponseWriter, r *http.Request) {
220 http.Error(w, "Error", http.StatusInternalServerError)
224 // Point keepproxy router's keepclient to the fake keepstore
225 sr := map[string]string{
226 TestProxyUUID: ts.URL,
228 router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
230 // Set up client to ask for storage classes to keepproxy
231 kc.StorageClasses = []string{"secure"}
232 content := []byte("Very important data")
233 _, _, err := kc.PutB(content)
235 c.Check(hdr.Get("X-Keep-Storage-Classes"), Equals, "secure")
238 func (s *ServerRequiredSuite) TestStorageClassesConfirmedHeader(c *C) {
239 runProxy(c, false, false, nil)
240 defer closeListener()
242 content := []byte("foo")
243 hash := fmt.Sprintf("%x", md5.Sum(content))
244 client := &http.Client{}
246 req, err := http.NewRequest("PUT",
247 fmt.Sprintf("http://%s/%s", listener.Addr().String(), hash),
248 bytes.NewReader(content))
250 req.Header.Set("X-Keep-Storage-Classes", "default")
251 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
252 req.Header.Set("Content-Type", "application/octet-stream")
254 resp, err := client.Do(req)
256 c.Assert(resp.StatusCode, Equals, http.StatusOK)
257 c.Assert(resp.Header.Get("X-Keep-Storage-Classes-Confirmed"), Equals, "default=2")
260 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
261 kc, _ := runProxy(c, false, false, nil)
262 defer closeListener()
264 content := []byte("TestDesiredReplicas")
265 hash := fmt.Sprintf("%x", md5.Sum(content))
267 for _, kc.Want_replicas = range []int{0, 1, 2, 3} {
268 locator, rep, err := kc.PutB(content)
269 if kc.Want_replicas < 3 {
270 c.Check(err, Equals, nil)
271 c.Check(rep, Equals, kc.Want_replicas)
273 c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
276 c.Check(err, ErrorMatches, ".*503.*")
281 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
282 kc, _ := runProxy(c, false, false, nil)
283 defer closeListener()
285 content := []byte("TestPutWrongContentLength")
286 hash := fmt.Sprintf("%x", md5.Sum(content))
288 // If we use http.Client to send these requests to the network
289 // server we just started, the Go http library automatically
290 // fixes the invalid Content-Length header. In order to test
291 // our server behavior, we have to call the handler directly
292 // using an httptest.ResponseRecorder.
293 rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{}, log.New())
294 c.Assert(err, check.IsNil)
296 type testcase struct {
301 for _, t := range []testcase{
302 {"1", http.StatusBadRequest},
303 {"", http.StatusLengthRequired},
304 {"-1", http.StatusLengthRequired},
305 {"abcdef", http.StatusLengthRequired},
307 req, err := http.NewRequest("PUT",
308 fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
309 bytes.NewReader(content))
311 req.Header.Set("Content-Length", t.sendLength)
312 req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
313 req.Header.Set("Content-Type", "application/octet-stream")
315 resp := httptest.NewRecorder()
316 rtr.ServeHTTP(resp, req)
317 c.Check(resp.Code, Equals, t.expectStatus)
321 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
322 kc, _ := runProxy(c, false, false, nil)
323 defer closeListener()
324 router.(*proxyHandler).timeout = time.Nanosecond
326 buf := make([]byte, 1<<20)
328 var wg sync.WaitGroup
329 for i := 0; i < 128; i++ {
336 done := make(chan bool)
343 case <-time.After(10 * time.Second):
348 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
349 kc, logbuf := runProxy(c, false, false, nil)
350 defer closeListener()
352 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
356 _, _, err := kc.Ask(hash)
357 c.Check(err, Equals, keepclient.BlockNotFound)
358 c.Log("Finished Ask (expected BlockNotFound)")
362 reader, _, _, err := kc.Get(hash)
363 c.Check(reader, Equals, nil)
364 c.Check(err, Equals, keepclient.BlockNotFound)
365 c.Log("Finished Get (expected BlockNotFound)")
368 // Note in bug #5309 among other errors keepproxy would set
369 // Content-Length incorrectly on the 404 BlockNotFound response, this
370 // would result in a protocol violation that would prevent reuse of the
371 // connection, which would manifest by the next attempt to use the
372 // connection (in this case the PutB below) failing. So to test for
373 // that bug it's necessary to trigger an error response (such as
374 // BlockNotFound) and then do something else with the same httpClient
380 hash2, rep, err = kc.PutB([]byte("foo"))
381 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
382 c.Check(rep, Equals, 2)
383 c.Check(err, Equals, nil)
384 c.Log("Finished PutB (expected success)")
386 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
391 blocklen, _, err := kc.Ask(hash2)
392 c.Assert(err, Equals, nil)
393 c.Check(blocklen, Equals, int64(3))
394 c.Log("Finished Ask (expected success)")
395 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
400 reader, blocklen, _, err := kc.Get(hash2)
401 c.Assert(err, Equals, nil)
402 all, err := ioutil.ReadAll(reader)
404 c.Check(all, DeepEquals, []byte("foo"))
405 c.Check(blocklen, Equals, int64(3))
406 c.Log("Finished Get (expected success)")
407 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
414 hash2, rep, err = kc.PutB([]byte(""))
415 c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
416 c.Check(rep, Equals, 2)
417 c.Check(err, Equals, nil)
418 c.Log("Finished PutB zero block")
422 reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
423 c.Assert(err, Equals, nil)
424 all, err := ioutil.ReadAll(reader)
426 c.Check(all, DeepEquals, []byte(""))
427 c.Check(blocklen, Equals, int64(0))
428 c.Log("Finished Get zero block")
432 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
433 kc, _ := runProxy(c, true, false, nil)
434 defer closeListener()
436 hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
438 _, _, err := kc.Ask(hash)
439 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
441 hash2, rep, err := kc.PutB([]byte("bar"))
442 c.Check(hash2, Equals, "")
443 c.Check(rep, Equals, 0)
444 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError{})
446 blocklen, _, err := kc.Ask(hash)
447 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
448 c.Check(err, ErrorMatches, ".*HTTP 403.*")
449 c.Check(blocklen, Equals, int64(0))
451 _, blocklen, _, err = kc.Get(hash)
452 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
453 c.Check(err, ErrorMatches, ".*HTTP 403.*")
454 c.Check(blocklen, Equals, int64(0))
457 func testPermission(c *C, admin bool, perm arvados.UploadDownloadPermission) {
458 kp := arvados.UploadDownloadRolePermissions{}
461 kp.User = arvados.UploadDownloadPermission{Upload: true, Download: true}
463 kp.Admin = arvados.UploadDownloadPermission{Upload: true, Download: true}
467 kc, logbuf := runProxy(c, false, false, &kp)
468 defer closeListener()
470 kc.Arvados.ApiToken = arvadostest.AdminToken
472 kc.Arvados.ApiToken = arvadostest.ActiveToken
475 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
481 hash2, rep, err = kc.PutB([]byte("foo"))
484 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
485 c.Check(rep, Equals, 2)
486 c.Check(err, Equals, nil)
487 c.Log("Finished PutB (expected success)")
489 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
492 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
495 c.Check(hash2, Equals, "")
496 c.Check(rep, Equals, 0)
497 c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError{})
502 // can't test download without upload.
504 reader, blocklen, _, err := kc.Get(hash2)
506 c.Assert(err, Equals, nil)
507 all, err := ioutil.ReadAll(reader)
509 c.Check(all, DeepEquals, []byte("foo"))
510 c.Check(blocklen, Equals, int64(3))
511 c.Log("Finished Get (expected success)")
513 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
515 c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
518 c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
519 c.Check(err, ErrorMatches, ".*Missing or invalid Authorization header, or method not allowed.*")
520 c.Check(blocklen, Equals, int64(0))
527 func (s *ServerRequiredSuite) TestPutGetPermission(c *C) {
529 for _, adminperm := range []bool{true, false} {
530 for _, userperm := range []bool{true, false} {
532 testPermission(c, true,
533 arvados.UploadDownloadPermission{
537 testPermission(c, true,
538 arvados.UploadDownloadPermission{
542 testPermission(c, false,
543 arvados.UploadDownloadPermission{
547 testPermission(c, false,
548 arvados.UploadDownloadPermission{
556 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
557 runProxy(c, false, false, nil)
558 defer closeListener()
561 client := http.Client{}
562 req, err := http.NewRequest("OPTIONS",
563 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
566 req.Header.Add("Access-Control-Request-Method", "PUT")
567 req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
568 resp, err := client.Do(req)
569 c.Check(err, Equals, nil)
570 c.Check(resp.StatusCode, Equals, 200)
571 body, err := ioutil.ReadAll(resp.Body)
573 c.Check(string(body), Equals, "")
574 c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
575 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
579 resp, err := http.Get(
580 fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
581 c.Check(err, Equals, nil)
582 c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
583 c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
587 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
588 runProxy(c, false, false, nil)
589 defer closeListener()
592 client := http.Client{}
593 req, err := http.NewRequest("POST",
594 "http://"+listener.Addr().String()+"/",
595 strings.NewReader("qux"))
597 req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
598 req.Header.Add("Content-Type", "application/octet-stream")
599 resp, err := client.Do(req)
600 c.Check(err, Equals, nil)
601 body, err := ioutil.ReadAll(resp.Body)
602 c.Check(err, Equals, nil)
603 c.Check(string(body), Matches,
604 fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
608 func (s *ServerRequiredSuite) TestStripHint(c *C) {
609 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
611 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
612 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
614 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
615 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
617 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
618 c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
620 "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
625 // Put one block, with 2 replicas
626 // With no prefix (expect the block locator, twice)
627 // With an existing prefix (expect the block locator, twice)
628 // With a valid but non-existing prefix (expect "\n")
629 // With an invalid prefix (expect error)
630 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
631 getIndexWorker(c, false)
636 // Put one block, with 2 replicas
637 // With no prefix (expect the block locator, twice)
638 // With an existing prefix (expect the block locator, twice)
639 // With a valid but non-existing prefix (expect "\n")
640 // With an invalid prefix (expect error)
641 func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
642 getIndexWorker(c, true)
645 func getIndexWorker(c *C, useConfig bool) {
646 kc, _ := runProxy(c, false, useConfig, nil)
647 defer closeListener()
649 // Put "index-data" blocks
650 data := []byte("index-data")
651 hash := fmt.Sprintf("%x", md5.Sum(data))
653 hash2, rep, err := kc.PutB(data)
654 c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
655 c.Check(rep, Equals, 2)
656 c.Check(err, Equals, nil)
658 reader, blocklen, _, err := kc.Get(hash)
660 c.Check(blocklen, Equals, int64(10))
661 all, err := ioutil.ReadAll(reader)
663 c.Check(all, DeepEquals, data)
665 // Put some more blocks
666 _, _, err = kc.PutB([]byte("some-more-index-data"))
669 kc.Arvados.ApiToken = arvadostest.SystemRootToken
672 for _, spec := range []struct {
677 {"", true, true}, // with no prefix
678 {hash[:3], true, false}, // with matching prefix
679 {"abcdef", false, false}, // with no such prefix
681 indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
682 c.Assert(err, Equals, nil)
683 indexResp, err := ioutil.ReadAll(indexReader)
684 c.Assert(err, Equals, nil)
685 locators := strings.Split(string(indexResp), "\n")
688 for _, locator := range locators {
692 c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
693 if locator[:32] == hash {
699 c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
700 c.Check(gotOther > 0, Equals, spec.expectOther)
703 // GetIndex with invalid prefix
704 _, err = kc.GetIndex(TestProxyUUID, "xyz")
705 c.Assert((err != nil), Equals, true)
708 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
709 kc, _ := runProxy(c, false, false, nil)
710 defer closeListener()
711 hash, _, err := kc.PutB([]byte("shareddata"))
713 kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
714 rdr, _, _, err := kc.Get(hash)
716 data, err := ioutil.ReadAll(rdr)
718 c.Check(data, DeepEquals, []byte("shareddata"))
721 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
722 kc, _ := runProxy(c, false, false, nil)
723 defer closeListener()
726 hash, rep, err := kc.PutB([]byte("foo"))
728 c.Check(rep, Equals, 2)
730 for _, badToken := range []string{
732 "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
734 kc.Arvados.ApiToken = badToken
736 // Ask and Get will fail only if the upstream
737 // keepstore server checks for valid signatures.
738 // Without knowing the blob signing key, there is no
739 // way for keepproxy to know whether a given token is
740 // permitted to read a block. So these tests fail:
742 _, _, err = kc.Ask(hash)
743 c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
744 c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
745 c.Check(err, ErrorMatches, ".*HTTP 403.*")
747 _, _, _, err = kc.Get(hash)
748 c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
749 c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
750 c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header, or method not allowed\".*")
753 _, _, err = kc.PutB([]byte("foo"))
754 c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header, or method not allowed")
758 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
759 kc, _ := runProxy(c, false, false, nil)
760 defer closeListener()
762 // Point keepproxy at a non-existent keepstore
763 locals := map[string]string{
764 TestProxyUUID: "http://localhost:12345",
766 router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
768 // Ask should result in temporary bad gateway error
769 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
770 _, _, err := kc.Ask(hash)
772 errNotFound, _ := err.(*keepclient.ErrNotFound)
773 c.Check(errNotFound.Temporary(), Equals, true)
774 c.Assert(err, ErrorMatches, ".*HTTP 502.*")
776 // Get should result in temporary bad gateway error
777 _, _, _, err = kc.Get(hash)
779 errNotFound, _ = err.(*keepclient.ErrNotFound)
780 c.Check(errNotFound.Temporary(), Equals, true)
781 c.Assert(err, ErrorMatches, ".*HTTP 502.*")
784 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
785 kc, _ := runProxy(c, false, false, nil)
786 defer closeListener()
788 hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
789 for _, f := range []func() error{
791 _, _, err := kc.Ask(hash)
795 _, _, _, err := kc.Get(hash)
800 c.Assert(err, NotNil)
801 errNotFound, _ := err.(*keepclient.ErrNotFound)
802 c.Check(errNotFound.Temporary(), Equals, true)
803 c.Check(err, ErrorMatches, `.*HTTP 502.*`)
807 func (s *ServerRequiredSuite) TestPing(c *C) {
808 kc, _ := runProxy(c, false, false, nil)
809 defer closeListener()
811 rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{ManagementToken: arvadostest.ManagementToken}, log.New())
812 c.Assert(err, check.IsNil)
814 req, err := http.NewRequest("GET",
815 "http://"+listener.Addr().String()+"/_health/ping",
818 req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
820 resp := httptest.NewRecorder()
821 rtr.ServeHTTP(resp, req)
822 c.Check(resp.Code, Equals, 200)
823 c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)