X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0e9443678fc7fd4862b76a10184dc7357a75155d..1bff2ab0181be31492c53351afc1c3c1e58ea05d:/services/keepproxy/keepproxy_test.go

diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index 8acf43abd4..65e22e3b3e 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -1,22 +1,29 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
+	"bytes"
 	"crypto/md5"
-	"crypto/tls"
+	"errors"
 	"fmt"
-	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
-	. "gopkg.in/check.v1"
-	"io"
 	"io/ioutil"
-	"log"
+	"math/rand"
 	"net/http"
-	"net/url"
+	"net/http/httptest"
 	"os"
-	"os/exec"
 	"strings"
+	"sync"
 	"testing"
 	"time"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+
+	. "gopkg.in/check.v1"
 )
 
 // Gocheck boilerplate
@@ -30,10 +37,13 @@ var _ = Suite(&ServerRequiredSuite{})
 // Tests that require the Keep server running
 type ServerRequiredSuite struct{}
 
-func pythonDir() string {
-	cwd, _ := os.Getwd()
-	return fmt.Sprintf("%s/../../sdk/python/tests", cwd)
-}
+// Gocheck boilerplate
+var _ = Suite(&NoKeepServerSuite{})
+
+// Test with no keepserver to simulate errors
+type NoKeepServerSuite struct{}
+
+var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
 
 // Wait (up to 1 second) for keepproxy to listen on a port. This
 // avoids a race condition where we hit a "connection refused" error
@@ -42,11 +52,11 @@ func waitForListener() {
 	const (
 		ms = 5
 	)
-	for i := 0; listener == nil && i < 1000; i += ms {
+	for i := 0; listener == nil && i < 10000; i += ms {
 		time.Sleep(ms * time.Millisecond)
 	}
 	if listener == nil {
-		log.Fatalf("Timed out waiting for listener to start")
+		panic("Timed out waiting for listener to start")
 	}
 }
 
@@ -57,135 +67,186 @@ func closeListener() {
 }
 
 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
-	cwd, _ := os.Getwd()
-	defer os.Chdir(cwd)
-
-	os.Chdir(pythonDir())
-	{
-		cmd := exec.Command("python", "run_test_server.py", "start")
-		stderr, err := cmd.StderrPipe()
-		if err != nil {
-			log.Fatalf("Setting up stderr pipe: %s", err)
-		}
-		go io.Copy(os.Stderr, stderr)
-		if err := cmd.Run(); err != nil {
-			panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err))
-		}
-	}
-	{
-		cmd := exec.Command("python", "run_test_server.py", "start_keep")
-		stderr, err := cmd.StderrPipe()
-		if err != nil {
-			log.Fatalf("Setting up stderr pipe: %s", err)
-		}
-		go io.Copy(os.Stderr, stderr)
-		if err := cmd.Run(); err != nil {
-			panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
-		}
-	}
+	arvadostest.StartAPI()
+	arvadostest.StartKeep(2, false)
+}
 
-	os.Setenv("ARVADOS_API_HOST", "localhost:3000")
-	os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
-	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+	arvadostest.ResetEnv()
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
-	cwd, _ := os.Getwd()
-	defer os.Chdir(cwd)
+	arvadostest.StopKeep(2)
+	arvadostest.StopAPI()
+}
+
+func (s *NoKeepServerSuite) SetUpSuite(c *C) {
+	arvadostest.StartAPI()
+	// We need API to have some keep services listed, but the
+	// services themselves should be unresponsive.
+	arvadostest.StartKeep(2, false)
+	arvadostest.StopKeep(2)
+}
 
-	os.Chdir(pythonDir())
-	exec.Command("python", "run_test_server.py", "stop_keep").Run()
-	exec.Command("python", "run_test_server.py", "stop").Run()
+func (s *NoKeepServerSuite) SetUpTest(c *C) {
+	arvadostest.ResetEnv()
 }
 
-func setupProxyService() {
+func (s *NoKeepServerSuite) TearDownSuite(c *C) {
+	arvadostest.StopAPI()
+}
 
-	client := &http.Client{Transport: &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient {
+	args = append([]string{"keepproxy"}, args...)
+	os.Args = append(args, "-listen=:0")
+	listener = nil
+	go main()
+	waitForListener()
 
-	var req *http.Request
-	var err error
-	if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
-		panic(err.Error())
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, Equals, nil)
+	if bogusClientToken {
+		arv.ApiToken = "bogus-token"
+	}
+	kc := keepclient.New(arv)
+	sr := map[string]string{
+		TestProxyUUID: "http://" + listener.Addr().String(),
 	}
-	req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
+	kc.SetServiceRoots(sr, sr, sr)
+	kc.Arvados.External = true
 
-	reader, writer := io.Pipe()
+	return kc
+}
 
-	req.Body = reader
+func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
+	runProxy(c, nil, false)
+	defer closeListener()
 
-	go func() {
-		data := url.Values{}
-		data.Set("keep_service", `{
-  "service_host": "localhost",
-  "service_port": 29950,
-  "service_ssl_flag": false,
-  "service_type": "proxy"
-}`)
-
-		writer.Write([]byte(data.Encode()))
-		writer.Close()
-	}()
+	req, err := http.NewRequest("POST",
+		"http://"+listener.Addr().String()+"/",
+		strings.NewReader("TestViaHeader"))
+	c.Assert(err, Equals, nil)
+	req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
+	resp, err := (&http.Client{}).Do(req)
+	c.Assert(err, Equals, nil)
+	c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
+	locator, err := ioutil.ReadAll(resp.Body)
+	c.Assert(err, Equals, nil)
+	resp.Body.Close()
 
-	var resp *http.Response
-	if resp, err = client.Do(req); err != nil {
-		panic(err.Error())
-	}
-	if resp.StatusCode != 200 {
-		panic(resp.Status)
+	req, err = http.NewRequest("GET",
+		"http://"+listener.Addr().String()+"/"+string(locator),
+		nil)
+	c.Assert(err, Equals, nil)
+	resp, err = (&http.Client{}).Do(req)
+	c.Assert(err, Equals, nil)
+	c.Check(resp.Header.Get("Via"), Equals, "HTTP/1.1 keepproxy")
+	resp.Body.Close()
+}
+
+func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
+
+	sr := map[string]string{
+		TestProxyUUID: "http://" + listener.Addr().String(),
 	}
+	router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
+
+	content := []byte("TestLoopDetection")
+	_, _, err := kc.PutB(content)
+	c.Check(err, ErrorMatches, `.*loop detected.*`)
+
+	hash := fmt.Sprintf("%x", md5.Sum(content))
+	_, _, _, err = kc.Get(hash)
+	c.Check(err, ErrorMatches, `.*loop detected.*`)
 }
 
-func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient {
-	os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
-	os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
 
-	listener = nil
-	go main()
-	time.Sleep(100 * time.Millisecond)
+	content := []byte("TestDesiredReplicas")
+	hash := fmt.Sprintf("%x", md5.Sum(content))
 
-	os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port))
-	os.Setenv("ARVADOS_API_TOKEN", token)
-	arv, err := arvadosclient.MakeArvadosClient()
-	c.Assert(err, Equals, nil)
-	kc, err := keepclient.MakeKeepClient(&arv)
-	c.Assert(err, Equals, nil)
-	c.Check(kc.Using_proxy, Equals, true)
-	c.Check(len(kc.ServiceRoots()), Equals, 1)
-	for _, root := range kc.ServiceRoots() {
-		c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port))
+	for _, kc.Want_replicas = range []int{0, 1, 2} {
+		locator, rep, err := kc.PutB(content)
+		c.Check(err, Equals, nil)
+		c.Check(rep, Equals, kc.Want_replicas)
+		if rep > 0 {
+			c.Check(locator, Matches, fmt.Sprintf(`^%s\+%d(\+.+)?$`, hash, len(content)))
+		}
 	}
-	os.Setenv("ARVADOS_KEEP_PROXY", "")
-	log.Print("keepclient created")
-	return kc
 }
 
-func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
-	log.Print("TestPutAndGet start")
+func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
 
-	os.Args = []string{"keepproxy", "-listen=:29950"}
-	os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
-	listener = nil
-	go main()
-	time.Sleep(100 * time.Millisecond)
+	content := []byte("TestPutWrongContentLength")
+	hash := fmt.Sprintf("%x", md5.Sum(content))
 
-	setupProxyService()
+	// If we use http.Client to send these requests to the network
+	// server we just started, the Go http library automatically
+	// fixes the invalid Content-Length header. In order to test
+	// our server behavior, we have to call the handler directly
+	// using an httptest.ResponseRecorder.
+	rtr := MakeRESTRouter(true, true, kc, 10*time.Second, "")
 
-	os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
-	arv, err := arvadosclient.MakeArvadosClient()
-	c.Assert(err, Equals, nil)
-	kc, err := keepclient.MakeKeepClient(&arv)
-	c.Assert(err, Equals, nil)
-	c.Check(kc.Arvados.External, Equals, true)
-	c.Check(kc.Using_proxy, Equals, true)
-	c.Check(len(kc.ServiceRoots()), Equals, 1)
-	for _, root := range kc.ServiceRoots() {
-		c.Check(root, Equals, "http://localhost:29950")
+	type testcase struct {
+		sendLength   string
+		expectStatus int
 	}
-	os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
-	log.Print("keepclient created")
 
-	waitForListener()
+	for _, t := range []testcase{
+		{"1", http.StatusBadRequest},
+		{"", http.StatusLengthRequired},
+		{"-1", http.StatusLengthRequired},
+		{"abcdef", http.StatusLengthRequired},
+	} {
+		req, err := http.NewRequest("PUT",
+			fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
+			bytes.NewReader(content))
+		c.Assert(err, IsNil)
+		req.Header.Set("Content-Length", t.sendLength)
+		req.Header.Set("Authorization", "OAuth2 "+arvadostest.ActiveToken)
+		req.Header.Set("Content-Type", "application/octet-stream")
+
+		resp := httptest.NewRecorder()
+		rtr.ServeHTTP(resp, req)
+		c.Check(resp.Code, Equals, t.expectStatus)
+	}
+}
+
+func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
+	router.(*proxyHandler).timeout = time.Nanosecond
+
+	buf := make([]byte, 1<<20)
+	rand.Read(buf)
+	var wg sync.WaitGroup
+	for i := 0; i < 128; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			kc.PutB(buf)
+		}()
+	}
+	done := make(chan bool)
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		c.Error("timeout")
+	}
+}
+
+func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
+	kc := runProxy(c, nil, false)
 	defer closeListener()
 
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
@@ -194,9 +255,25 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 	{
 		_, _, err := kc.Ask(hash)
 		c.Check(err, Equals, keepclient.BlockNotFound)
-		log.Print("Ask 1")
+		c.Log("Finished Ask (expected BlockNotFound)")
+	}
+
+	{
+		reader, _, _, err := kc.Get(hash)
+		c.Check(reader, Equals, nil)
+		c.Check(err, Equals, keepclient.BlockNotFound)
+		c.Log("Finished Get (expected BlockNotFound)")
 	}
 
+	// Note in bug #5309 among other errors keepproxy would set
+	// Content-Length incorrectly on the 404 BlockNotFound response, this
+	// would result in a protocol violation that would prevent reuse of the
+	// connection, which would manifest by the next attempt to use the
+	// connection (in this case the PutB below) failing.  So to test for
+	// that bug it's necessary to trigger an error response (such as
+	// BlockNotFound) and then do something else with the same httpClient
+	// connection.
+
 	{
 		var rep int
 		var err error
@@ -204,23 +281,24 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 		c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 		c.Check(rep, Equals, 2)
 		c.Check(err, Equals, nil)
-		log.Print("PutB")
+		c.Log("Finished PutB (expected success)")
 	}
 
 	{
 		blocklen, _, err := kc.Ask(hash2)
 		c.Assert(err, Equals, nil)
 		c.Check(blocklen, Equals, int64(3))
-		log.Print("Ask 2")
+		c.Log("Finished Ask (expected success)")
 	}
 
 	{
 		reader, blocklen, _, err := kc.Get(hash2)
 		c.Assert(err, Equals, nil)
 		all, err := ioutil.ReadAll(reader)
+		c.Check(err, IsNil)
 		c.Check(all, DeepEquals, []byte("foo"))
 		c.Check(blocklen, Equals, int64(3))
-		log.Print("Get")
+		c.Log("Finished Get (expected success)")
 	}
 
 	{
@@ -230,76 +308,58 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 		c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
 		c.Check(rep, Equals, 2)
 		c.Check(err, Equals, nil)
-		log.Print("PutB zero block")
+		c.Log("Finished PutB zero block")
 	}
 
 	{
 		reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
 		c.Assert(err, Equals, nil)
 		all, err := ioutil.ReadAll(reader)
+		c.Check(err, IsNil)
 		c.Check(all, DeepEquals, []byte(""))
 		c.Check(blocklen, Equals, int64(0))
-		log.Print("Get zero block")
+		c.Log("Finished Get zero block")
 	}
-
-	log.Print("TestPutAndGet done")
 }
 
 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
-	log.Print("TestPutAskGetForbidden start")
-
-	kc := runProxy(c, []string{"keepproxy"}, "123abc", 29951)
-	waitForListener()
+	kc := runProxy(c, nil, true)
 	defer closeListener()
 
-	log.Print("keepclient created")
+	hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
 
-	hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
+	_, _, err := kc.Ask(hash)
+	c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
 
-	{
-		_, _, err := kc.Ask(hash)
-		c.Check(err, Equals, keepclient.BlockNotFound)
-		log.Print("Ask 1")
-	}
+	hash2, rep, err := kc.PutB([]byte("bar"))
+	c.Check(hash2, Equals, "")
+	c.Check(rep, Equals, 0)
+	c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 
-	{
-		hash2, rep, err := kc.PutB([]byte("bar"))
-		c.Check(hash2, Equals, "")
-		c.Check(rep, Equals, 0)
-		c.Check(err, Equals, keepclient.InsufficientReplicasError)
-		log.Print("PutB")
-	}
+	blocklen, _, err := kc.Ask(hash)
+	c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
+	c.Check(err, ErrorMatches, ".*not found.*")
+	c.Check(blocklen, Equals, int64(0))
 
-	{
-		blocklen, _, err := kc.Ask(hash)
-		c.Assert(err, Equals, keepclient.BlockNotFound)
-		c.Check(blocklen, Equals, int64(0))
-		log.Print("Ask 2")
-	}
+	_, blocklen, _, err = kc.Get(hash)
+	c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
+	c.Check(err, ErrorMatches, ".*not found.*")
+	c.Check(blocklen, Equals, int64(0))
 
-	{
-		_, blocklen, _, err := kc.Get(hash)
-		c.Assert(err, Equals, keepclient.BlockNotFound)
-		c.Check(blocklen, Equals, int64(0))
-		log.Print("Get")
-	}
-
-	log.Print("TestPutAskGetForbidden done")
 }
 
 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
-	log.Print("TestGetDisabled start")
-
-	kc := runProxy(c, []string{"keepproxy", "-no-get"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29952)
-	waitForListener()
+	kc := runProxy(c, []string{"-no-get"}, false)
 	defer closeListener()
 
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 
 	{
 		_, _, err := kc.Ask(hash)
-		c.Check(err, Equals, keepclient.BlockNotFound)
-		log.Print("Ask 1")
+		errNotFound, _ := err.(keepclient.ErrNotFound)
+		c.Check(errNotFound, NotNil)
+		c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
+		c.Log("Ask 1")
 	}
 
 	{
@@ -307,61 +367,55 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 		c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
 		c.Check(rep, Equals, 2)
 		c.Check(err, Equals, nil)
-		log.Print("PutB")
+		c.Log("PutB")
 	}
 
 	{
 		blocklen, _, err := kc.Ask(hash)
-		c.Assert(err, Equals, keepclient.BlockNotFound)
+		errNotFound, _ := err.(keepclient.ErrNotFound)
+		c.Check(errNotFound, NotNil)
+		c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 		c.Check(blocklen, Equals, int64(0))
-		log.Print("Ask 2")
+		c.Log("Ask 2")
 	}
 
 	{
 		_, blocklen, _, err := kc.Get(hash)
-		c.Assert(err, Equals, keepclient.BlockNotFound)
+		errNotFound, _ := err.(keepclient.ErrNotFound)
+		c.Check(errNotFound, NotNil)
+		c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
 		c.Check(blocklen, Equals, int64(0))
-		log.Print("Get")
+		c.Log("Get")
 	}
-
-	log.Print("TestGetDisabled done")
 }
 
 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
-	log.Print("TestPutDisabled start")
-
-	kc := runProxy(c, []string{"keepproxy", "-no-put"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29953)
-	waitForListener()
+	kc := runProxy(c, []string{"-no-put"}, false)
 	defer closeListener()
 
-	{
-		hash2, rep, err := kc.PutB([]byte("quux"))
-		c.Check(hash2, Equals, "")
-		c.Check(rep, Equals, 0)
-		c.Check(err, Equals, keepclient.InsufficientReplicasError)
-		log.Print("PutB")
-	}
-
-	log.Print("TestPutDisabled done")
+	hash2, rep, err := kc.PutB([]byte("quux"))
+	c.Check(hash2, Equals, "")
+	c.Check(rep, Equals, 0)
+	c.Check(err, FitsTypeOf, keepclient.InsufficientReplicasError(errors.New("")))
 }
 
 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
-	runProxy(c, []string{"keepproxy"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29954)
-	waitForListener()
+	runProxy(c, nil, false)
 	defer closeListener()
 
 	{
 		client := http.Client{}
 		req, err := http.NewRequest("OPTIONS",
-			fmt.Sprintf("http://localhost:29954/%x+3",
-				md5.Sum([]byte("foo"))),
+			fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
 			nil)
+		c.Assert(err, IsNil)
 		req.Header.Add("Access-Control-Request-Method", "PUT")
 		req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
 		resp, err := client.Do(req)
 		c.Check(err, Equals, nil)
 		c.Check(resp.StatusCode, Equals, 200)
 		body, err := ioutil.ReadAll(resp.Body)
+		c.Check(err, IsNil)
 		c.Check(string(body), Equals, "")
 		c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
 		c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
@@ -369,8 +423,7 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 
 	{
 		resp, err := http.Get(
-			fmt.Sprintf("http://localhost:29954/%x+3",
-				md5.Sum([]byte("foo"))))
+			fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
 		c.Check(err, Equals, nil)
 		c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
 		c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
@@ -378,22 +431,225 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
-	runProxy(c, []string{"keepproxy"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29955)
-	waitForListener()
+	runProxy(c, nil, false)
 	defer closeListener()
 
 	{
 		client := http.Client{}
 		req, err := http.NewRequest("POST",
-			"http://localhost:29955/",
+			"http://"+listener.Addr().String()+"/",
 			strings.NewReader("qux"))
-		req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+		c.Check(err, IsNil)
+		req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
 		req.Header.Add("Content-Type", "application/octet-stream")
 		resp, err := client.Do(req)
 		c.Check(err, Equals, nil)
 		body, err := ioutil.ReadAll(resp.Body)
 		c.Check(err, Equals, nil)
-		c.Check(string(body), Equals,
-			fmt.Sprintf("%x+%d", md5.Sum([]byte("qux")), 3))
+		c.Check(string(body), Matches,
+			fmt.Sprintf(`^%x\+3(\+.+)?$`, md5.Sum([]byte("qux"))))
 	}
 }
+
+func (s *ServerRequiredSuite) TestStripHint(c *C) {
+	c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
+		Equals,
+		"http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
+	c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
+		Equals,
+		"http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
+	c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
+		Equals,
+		"http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+	c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
+		Equals,
+		"http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
+
+}
+
+// Test GetIndex
+//   Put one block, with 2 replicas
+//   With no prefix (expect the block locator, twice)
+//   With an existing prefix (expect the block locator, twice)
+//   With a valid but non-existing prefix (expect "\n")
+//   With an invalid prefix (expect error)
+func (s *ServerRequiredSuite) TestGetIndex(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
+
+	// Put "index-data" blocks
+	data := []byte("index-data")
+	hash := fmt.Sprintf("%x", md5.Sum(data))
+
+	hash2, rep, err := kc.PutB(data)
+	c.Check(hash2, Matches, fmt.Sprintf(`^%s\+10(\+.+)?$`, hash))
+	c.Check(rep, Equals, 2)
+	c.Check(err, Equals, nil)
+
+	reader, blocklen, _, err := kc.Get(hash)
+	c.Assert(err, IsNil)
+	c.Check(blocklen, Equals, int64(10))
+	all, err := ioutil.ReadAll(reader)
+	c.Assert(err, IsNil)
+	c.Check(all, DeepEquals, data)
+
+	// Put some more blocks
+	_, _, err = kc.PutB([]byte("some-more-index-data"))
+	c.Check(err, IsNil)
+
+	kc.Arvados.ApiToken = arvadostest.DataManagerToken
+
+	// Invoke GetIndex
+	for _, spec := range []struct {
+		prefix         string
+		expectTestHash bool
+		expectOther    bool
+	}{
+		{"", true, true},         // with no prefix
+		{hash[:3], true, false},  // with matching prefix
+		{"abcdef", false, false}, // with no such prefix
+	} {
+		indexReader, err := kc.GetIndex(TestProxyUUID, spec.prefix)
+		c.Assert(err, Equals, nil)
+		indexResp, err := ioutil.ReadAll(indexReader)
+		c.Assert(err, Equals, nil)
+		locators := strings.Split(string(indexResp), "\n")
+		gotTestHash := 0
+		gotOther := 0
+		for _, locator := range locators {
+			if locator == "" {
+				continue
+			}
+			c.Check(locator[:len(spec.prefix)], Equals, spec.prefix)
+			if locator[:32] == hash {
+				gotTestHash++
+			} else {
+				gotOther++
+			}
+		}
+		c.Check(gotTestHash == 2, Equals, spec.expectTestHash)
+		c.Check(gotOther > 0, Equals, spec.expectOther)
+	}
+
+	// GetIndex with invalid prefix
+	_, err = kc.GetIndex(TestProxyUUID, "xyz")
+	c.Assert((err != nil), Equals, true)
+}
+
+func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
+	hash, _, err := kc.PutB([]byte("shareddata"))
+	c.Check(err, IsNil)
+	kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
+	rdr, _, _, err := kc.Get(hash)
+	c.Assert(err, IsNil)
+	data, err := ioutil.ReadAll(rdr)
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("shareddata"))
+}
+
+func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
+
+	// Put a test block
+	hash, rep, err := kc.PutB([]byte("foo"))
+	c.Check(err, IsNil)
+	c.Check(rep, Equals, 2)
+
+	for _, badToken := range []string{
+		"nosuchtoken",
+		"2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired
+	} {
+		kc.Arvados.ApiToken = badToken
+
+		// Ask and Get will fail only if the upstream
+		// keepstore server checks for valid signatures.
+		// Without knowing the blob signing key, there is no
+		// way for keepproxy to know whether a given token is
+		// permitted to read a block.  So these tests fail:
+		if false {
+			_, _, err = kc.Ask(hash)
+			c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
+			c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
+			c.Check(err, ErrorMatches, ".*HTTP 403.*")
+
+			_, _, _, err = kc.Get(hash)
+			c.Assert(err, FitsTypeOf, &keepclient.ErrNotFound{})
+			c.Check(err.(*keepclient.ErrNotFound).Temporary(), Equals, false)
+			c.Check(err, ErrorMatches, ".*HTTP 403 \"Missing or invalid Authorization header\".*")
+		}
+
+		_, _, err = kc.PutB([]byte("foo"))
+		c.Check(err, ErrorMatches, ".*403.*Missing or invalid Authorization header")
+	}
+}
+
+func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, Equals, nil)
+
+	// keepclient with no such keep server
+	kc := keepclient.New(arv)
+	locals := map[string]string{
+		TestProxyUUID: "http://localhost:12345",
+	}
+	kc.SetServiceRoots(locals, nil, nil)
+
+	// Ask should result in temporary connection refused error
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+	_, _, err = kc.Ask(hash)
+	c.Check(err, NotNil)
+	errNotFound, _ := err.(*keepclient.ErrNotFound)
+	c.Check(errNotFound.Temporary(), Equals, true)
+	c.Assert(err, ErrorMatches, ".*connection refused.*")
+
+	// Get should result in temporary connection refused error
+	_, _, _, err = kc.Get(hash)
+	c.Check(err, NotNil)
+	errNotFound, _ = err.(*keepclient.ErrNotFound)
+	c.Check(errNotFound.Temporary(), Equals, true)
+	c.Assert(err, ErrorMatches, ".*connection refused.*")
+}
+
+func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
+
+	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+	for _, f := range []func() error{
+		func() error {
+			_, _, err := kc.Ask(hash)
+			return err
+		},
+		func() error {
+			_, _, _, err := kc.Get(hash)
+			return err
+		},
+	} {
+		err := f()
+		c.Assert(err, NotNil)
+		errNotFound, _ := err.(*keepclient.ErrNotFound)
+		c.Check(errNotFound.Temporary(), Equals, true)
+		c.Check(err, ErrorMatches, `.*HTTP 502.*`)
+	}
+}
+
+func (s *ServerRequiredSuite) TestPing(c *C) {
+	kc := runProxy(c, nil, false)
+	defer closeListener()
+
+	rtr := MakeRESTRouter(true, true, kc, 10*time.Second, arvadostest.ManagementToken)
+
+	req, err := http.NewRequest("GET",
+		"http://"+listener.Addr().String()+"/_health/ping",
+		nil)
+	c.Assert(err, IsNil)
+	req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
+
+	resp := httptest.NewRecorder()
+	rtr.ServeHTTP(resp, req)
+	c.Check(resp.Code, Equals, 200)
+	c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
+}