X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/e809173d733bc78ce7877f6ed9711b29fd7cee0e..f2019e7042d12088bce45f8c2ad52ec600a4076d:/services/keepproxy/keepproxy.go

diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 581f7f4873..3d1b447625 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -1,11 +1,13 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
-	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/keepclient"
-	"github.com/gorilla/mux"
 	"io"
 	"io/ioutil"
 	"log"
@@ -13,99 +15,141 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"regexp"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvados"
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/config"
+	"git.curoverse.com/arvados.git/sdk/go/health"
+	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+	"github.com/coreos/go-systemd/daemon"
+	"github.com/ghodss/yaml"
+	"github.com/gorilla/mux"
 )
 
-// Default TCP address on which to listen for requests.
-// Initialized by the -listen flag.
-const DEFAULT_ADDR = ":25107"
+type Config struct {
+	Client          arvados.Client
+	Listen          string
+	DisableGet      bool
+	DisablePut      bool
+	DefaultReplicas int
+	Timeout         arvados.Duration
+	PIDFile         string
+	Debug           bool
+	ManagementToken string
+}
+
+func DefaultConfig() *Config {
+	return &Config{
+		Listen:  ":25107",
+		Timeout: arvados.Duration(15 * time.Second),
+	}
+}
 
-var listener net.Listener
+var (
+	listener net.Listener
+	router   http.Handler
+)
 
 func main() {
-	var (
-		listen           string
-		no_get           bool
-		no_put           bool
-		default_replicas int
-		timeout          int64
-		pidfile          string
-	)
-
-	flagset := flag.NewFlagSet("default", flag.ExitOnError)
-
-	flagset.StringVar(
-		&listen,
-		"listen",
-		DEFAULT_ADDR,
-		"Interface on which to listen for requests, in the format "+
-			"ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port "+
-			"to listen on all network interfaces.")
-
-	flagset.BoolVar(
-		&no_get,
-		"no-get",
-		false,
-		"If set, disable GET operations")
-
-	flagset.BoolVar(
-		&no_put,
-		"no-put",
-		false,
-		"If set, disable PUT operations")
-
-	flagset.IntVar(
-		&default_replicas,
-		"default-replicas",
-		2,
-		"Default number of replicas to write if not specified by the client.")
-
-	flagset.Int64Var(
-		&timeout,
-		"timeout",
-		15,
-		"Timeout on requests to internal Keep services (default 15 seconds)")
-
-	flagset.StringVar(
-		&pidfile,
-		"pid",
-		"",
-		"Path to write pid file")
-
+	cfg := DefaultConfig()
+
+	flagset := flag.NewFlagSet("keepproxy", flag.ExitOnError)
+	flagset.Usage = usage
+
+	const deprecated = " (DEPRECATED -- use config file instead)"
+	flagset.StringVar(&cfg.Listen, "listen", cfg.Listen, "Local port to listen on."+deprecated)
+	flagset.BoolVar(&cfg.DisableGet, "no-get", cfg.DisableGet, "Disable GET operations."+deprecated)
+	flagset.BoolVar(&cfg.DisablePut, "no-put", cfg.DisablePut, "Disable PUT operations."+deprecated)
+	flagset.IntVar(&cfg.DefaultReplicas, "default-replicas", cfg.DefaultReplicas, "Default number of replicas to write if not specified by the client. If 0, use site default."+deprecated)
+	flagset.StringVar(&cfg.PIDFile, "pid", cfg.PIDFile, "Path to write pid file."+deprecated)
+	timeoutSeconds := flagset.Int("timeout", int(time.Duration(cfg.Timeout)/time.Second), "Timeout (in seconds) on requests to internal Keep services."+deprecated)
+	flagset.StringVar(&cfg.ManagementToken, "management-token", cfg.ManagementToken, "Authorization token to be included in all health check requests.")
+
+	var cfgPath string
+	const defaultCfgPath = "/etc/arvados/keepproxy/keepproxy.yml"
+	flagset.StringVar(&cfgPath, "config", defaultCfgPath, "Configuration file `path`")
+	dumpConfig := flagset.Bool("dump-config", false, "write current configuration to stdout and exit")
 	flagset.Parse(os.Args[1:])
 
-	arv, err := arvadosclient.MakeArvadosClient()
+	err := config.LoadFile(cfg, cfgPath)
+	if err != nil {
+		h := os.Getenv("ARVADOS_API_HOST")
+		t := os.Getenv("ARVADOS_API_TOKEN")
+		if h == "" || t == "" || !os.IsNotExist(err) || cfgPath != defaultCfgPath {
+			log.Fatal(err)
+		}
+		log.Print("DEPRECATED: No config file found, but ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are set. Please use a config file instead.")
+		cfg.Client.APIHost = h
+		cfg.Client.AuthToken = t
+		if regexp.MustCompile("^(?i:1|yes|true)$").MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) {
+			cfg.Client.Insecure = true
+		}
+		if y, err := yaml.Marshal(cfg); err == nil && !*dumpConfig {
+			log.Print("Current configuration:\n", string(y))
+		}
+		cfg.Timeout = arvados.Duration(time.Duration(*timeoutSeconds) * time.Second)
+	}
+
+	if *dumpConfig {
+		log.Fatal(config.DumpAndExit(cfg))
+	}
+
+	arv, err := arvadosclient.New(&cfg.Client)
 	if err != nil {
 		log.Fatalf("Error setting up arvados client %s", err.Error())
 	}
 
-	kc, err := keepclient.MakeKeepClient(&arv)
+	if cfg.Debug {
+		keepclient.DebugPrintf = log.Printf
+	}
+	kc, err := keepclient.MakeKeepClient(arv)
 	if err != nil {
 		log.Fatalf("Error setting up keep client %s", err.Error())
 	}
+	keepclient.RefreshServiceDiscoveryOnSIGHUP()
 
-	if pidfile != "" {
-		f, err := os.Create(pidfile)
+	if cfg.PIDFile != "" {
+		f, err := os.Create(cfg.PIDFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer f.Close()
+		err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+		if err != nil {
+			log.Fatalf("flock(%s): %s", cfg.PIDFile, err)
+		}
+		defer os.Remove(cfg.PIDFile)
+		err = f.Truncate(0)
+		if err != nil {
+			log.Fatalf("truncate(%s): %s", cfg.PIDFile, err)
+		}
+		_, err = fmt.Fprint(f, os.Getpid())
+		if err != nil {
+			log.Fatalf("write(%s): %s", cfg.PIDFile, err)
+		}
+		err = f.Sync()
 		if err != nil {
-			log.Fatalf("Error writing pid file (%s): %s", pidfile, err.Error())
+			log.Fatal("sync(%s): %s", cfg.PIDFile, err)
 		}
-		fmt.Fprint(f, os.Getpid())
-		f.Close()
-		defer os.Remove(pidfile)
 	}
 
-	kc.Want_replicas = default_replicas
-
-	kc.Client.Timeout = time.Duration(timeout) * time.Second
+	if cfg.DefaultReplicas > 0 {
+		kc.Want_replicas = cfg.DefaultReplicas
+	}
 
-	listener, err = net.Listen("tcp", listen)
+	listener, err = net.Listen("tcp", cfg.Listen)
 	if err != nil {
-		log.Fatalf("Could not listen on %v", listen)
+		log.Fatalf("listen(%s): %s", cfg.Listen, err)
 	}
-
-	go RefreshServicesList(&kc)
+	if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
+		log.Printf("Error notifying init daemon: %v", err)
+	}
+	log.Println("Listening at", listener.Addr())
 
 	// Shut down the server gracefully (by closing the listener)
 	// if SIGTERM is received.
@@ -118,10 +162,9 @@ func main() {
 	signal.Notify(term, syscall.SIGTERM)
 	signal.Notify(term, syscall.SIGINT)
 
-	log.Printf("Arvados Keep proxy started listening on %v with server list %v", listener.Addr(), kc.ServiceRoots())
-
-	// Start listening for requests.
-	http.Serve(listener, MakeRESTRouter(!no_get, !no_put, &kc))
+	// Start serving requests.
+	router = MakeRESTRouter(!cfg.DisableGet, !cfg.DisablePut, kc, time.Duration(cfg.Timeout), cfg.ManagementToken)
+	http.Serve(listener, router)
 
 	log.Println("shutting down")
 }
@@ -132,32 +175,6 @@ type ApiTokenCache struct {
 	expireTime int64
 }
 
-// Refresh the keep service list every five minutes.
-func RefreshServicesList(kc *keepclient.KeepClient) {
-	var sleeptime time.Duration
-	for {
-		oldservices := kc.ServiceRoots()
-		newservices, err := kc.DiscoverKeepServers()
-		if err == nil && len(newservices) > 0 {
-			s1 := fmt.Sprint(oldservices)
-			s2 := fmt.Sprint(newservices)
-			if s1 != s2 {
-				log.Printf("Updated server list to %v", s2)
-			}
-			sleeptime = 300 * time.Second
-		} else {
-			// There was an error, or the list is empty, so wait 3 seconds and try again.
-			if err != nil {
-				log.Printf("Error retrieving server list: %v", err)
-			} else {
-				log.Printf("Retrieved an empty server list")
-			}
-			sleeptime = 3 * time.Second
-		}
-		time.Sleep(sleeptime)
-	}
-}
-
 // Cache the token and set an expire time.  If we already have an expire time
 // on the token, it is not updated.
 func (this *ApiTokenCache) RememberToken(token string) {
@@ -190,17 +207,13 @@ func (this *ApiTokenCache) RecallToken(token string) bool {
 }
 
 func GetRemoteAddress(req *http.Request) string {
-	if realip := req.Header.Get("X-Real-IP"); realip != "" {
-		if forwarded := req.Header.Get("X-Forwarded-For"); forwarded != realip {
-			return fmt.Sprintf("%s (X-Forwarded-For %s)", realip, forwarded)
-		} else {
-			return realip
-		}
+	if xff := req.Header.Get("X-Forwarded-For"); xff != "" {
+		return xff + "," + req.RemoteAddr
 	}
 	return req.RemoteAddr
 }
 
-func CheckAuthorizationHeader(kc keepclient.KeepClient, cache *ApiTokenCache, req *http.Request) (pass bool, tok string) {
+func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *ApiTokenCache, req *http.Request) (pass bool, tok string) {
 	var auth string
 	if auth = req.Header.Get("Authorization"); auth == "" {
 		return false, ""
@@ -213,7 +226,7 @@ func CheckAuthorizationHeader(kc keepclient.KeepClient, cache *ApiTokenCache, re
 	}
 
 	if cache.RecallToken(tok) {
-		// Valid in the cache, short circut
+		// Valid in the cache, short circuit
 		return true, tok
 	}
 
@@ -230,50 +243,76 @@ func CheckAuthorizationHeader(kc keepclient.KeepClient, cache *ApiTokenCache, re
 	return true, tok
 }
 
-type GetBlockHandler struct {
+type proxyHandler struct {
+	http.Handler
 	*keepclient.KeepClient
 	*ApiTokenCache
+	timeout   time.Duration
+	transport *http.Transport
 }
 
-type PutBlockHandler struct {
-	*keepclient.KeepClient
-	*ApiTokenCache
-}
-
-type InvalidPathHandler struct{}
+// MakeRESTRouter returns an http.Handler that passes GET and PUT
+// requests to the appropriate handlers.
+func MakeRESTRouter(enable_get bool, enable_put bool, kc *keepclient.KeepClient, timeout time.Duration, mgmtToken string) http.Handler {
+	rest := mux.NewRouter()
 
-type OptionsHandler struct{}
+	transport := *(http.DefaultTransport.(*http.Transport))
+	transport.DialContext = (&net.Dialer{
+		Timeout:   keepclient.DefaultConnectTimeout,
+		KeepAlive: keepclient.DefaultKeepAlive,
+		DualStack: true,
+	}).DialContext
+	transport.TLSClientConfig = arvadosclient.MakeTLSConfig(kc.Arvados.ApiInsecure)
+	transport.TLSHandshakeTimeout = keepclient.DefaultTLSHandshakeTimeout
+
+	h := &proxyHandler{
+		Handler:    rest,
+		KeepClient: kc,
+		timeout:    timeout,
+		transport:  &transport,
+		ApiTokenCache: &ApiTokenCache{
+			tokens:     make(map[string]int64),
+			expireTime: 300,
+		},
+	}
 
-// MakeRESTRouter
-//     Returns a mux.Router that passes GET and PUT requests to the
-//     appropriate handlers.
-//
-func MakeRESTRouter(
-	enable_get bool,
-	enable_put bool,
-	kc *keepclient.KeepClient) *mux.Router {
+	if enable_get {
+		rest.HandleFunc(`/{locator:[0-9a-f]{32}\+.*}`, h.Get).Methods("GET", "HEAD")
+		rest.HandleFunc(`/{locator:[0-9a-f]{32}}`, h.Get).Methods("GET", "HEAD")
 
-	t := &ApiTokenCache{tokens: make(map[string]int64), expireTime: 300}
+		// List all blocks
+		rest.HandleFunc(`/index`, h.Index).Methods("GET")
 
-	rest := mux.NewRouter()
-
-	if enable_get {
-		rest.Handle(`/{hash:[0-9a-f]{32}}+{hints}`,
-			GetBlockHandler{kc, t}).Methods("GET", "HEAD")
-		rest.Handle(`/{hash:[0-9a-f]{32}}`, GetBlockHandler{kc, t}).Methods("GET", "HEAD")
+		// List blocks whose hash has the given prefix
+		rest.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, h.Index).Methods("GET")
 	}
 
 	if enable_put {
-		rest.Handle(`/{hash:[0-9a-f]{32}}+{hints}`, PutBlockHandler{kc, t}).Methods("PUT")
-		rest.Handle(`/{hash:[0-9a-f]{32}}`, PutBlockHandler{kc, t}).Methods("PUT")
-		rest.Handle(`/`, PutBlockHandler{kc, t}).Methods("POST")
-		rest.Handle(`/{any}`, OptionsHandler{}).Methods("OPTIONS")
-		rest.Handle(`/`, OptionsHandler{}).Methods("OPTIONS")
+		rest.HandleFunc(`/{locator:[0-9a-f]{32}\+.*}`, h.Put).Methods("PUT")
+		rest.HandleFunc(`/{locator:[0-9a-f]{32}}`, h.Put).Methods("PUT")
+		rest.HandleFunc(`/`, h.Put).Methods("POST")
+		rest.HandleFunc(`/{any}`, h.Options).Methods("OPTIONS")
+		rest.HandleFunc(`/`, h.Options).Methods("OPTIONS")
 	}
 
+	rest.Handle("/_health/{check}", &health.Handler{
+		Token:  mgmtToken,
+		Prefix: "/_health/",
+	}).Methods("GET")
+
 	rest.NotFoundHandler = InvalidPathHandler{}
+	return h
+}
 
-	return rest
+var errLoopDetected = errors.New("loop detected")
+
+func (*proxyHandler) checkLoop(resp http.ResponseWriter, req *http.Request) error {
+	if via := req.Header.Get("Via"); strings.Index(via, " "+viaAlias) >= 0 {
+		log.Printf("proxy loop detected (request has Via: %q): perhaps keepproxy is misidentified by gateway config as an external client, or its keep_services record does not have service_type=proxy?", via)
+		http.Error(resp, errLoopDetected.Error(), http.StatusInternalServerError)
+		return errLoopDetected
+	}
+	return nil
 }
 
 func SetCorsHeaders(resp http.ResponseWriter) {
@@ -283,32 +322,50 @@ func SetCorsHeaders(resp http.ResponseWriter) {
 	resp.Header().Set("Access-Control-Max-Age", "86486400")
 }
 
-func (this InvalidPathHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+type InvalidPathHandler struct{}
+
+func (InvalidPathHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 	log.Printf("%s: %s %s unroutable", GetRemoteAddress(req), req.Method, req.URL.Path)
 	http.Error(resp, "Bad request", http.StatusBadRequest)
 }
 
-func (this OptionsHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h *proxyHandler) Options(resp http.ResponseWriter, req *http.Request) {
 	log.Printf("%s: %s %s", GetRemoteAddress(req), req.Method, req.URL.Path)
 	SetCorsHeaders(resp)
 }
 
-func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-	SetCorsHeaders(resp)
+var BadAuthorizationHeader = errors.New("Missing or invalid Authorization header")
+var ContentLengthMismatch = errors.New("Actual length != expected content length")
+var MethodNotSupported = errors.New("Method not supported")
 
-	kc := *this.KeepClient
+var removeHint, _ = regexp.Compile("\\+K@[a-z0-9]{5}(\\+|$)")
 
-	hash := mux.Vars(req)["hash"]
-	hints := mux.Vars(req)["hints"]
+func (h *proxyHandler) Get(resp http.ResponseWriter, req *http.Request) {
+	if err := h.checkLoop(resp, req); err != nil {
+		return
+	}
+	SetCorsHeaders(resp)
+	resp.Header().Set("Via", req.Proto+" "+viaAlias)
 
-	locator := keepclient.MakeLocator2(hash, hints)
+	locator := mux.Vars(req)["locator"]
+	var err error
+	var status int
+	var expectLength, responseLength int64
+	var proxiedURI = "-"
+
+	defer func() {
+		log.Println(GetRemoteAddress(req), req.Method, req.URL.Path, status, expectLength, responseLength, proxiedURI, err)
+		if status != http.StatusOK {
+			http.Error(resp, err.Error(), status)
+		}
+	}()
 
-	log.Printf("%s: %s %s begin", GetRemoteAddress(req), req.Method, hash)
+	kc := h.makeKeepClient(req)
 
 	var pass bool
 	var tok string
-	if pass, tok = CheckAuthorizationHeader(kc, this.ApiTokenCache, req); !pass {
-		http.Error(resp, "Missing or invalid Authorization header", http.StatusForbidden)
+	if pass, tok = CheckAuthorizationHeader(kc, h.ApiTokenCache, req); !pass {
+		status, err = http.StatusForbidden, BadAuthorizationHeader
 		return
 	}
 
@@ -318,92 +375,103 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 	kc.Arvados = &arvclient
 
 	var reader io.ReadCloser
-	var err error
-	var blocklen int64
 
-	if req.Method == "GET" {
-		reader, blocklen, _, err = kc.AuthorizedGet(hash, locator.Signature, locator.Timestamp)
+	locator = removeHint.ReplaceAllString(locator, "$1")
+
+	switch req.Method {
+	case "HEAD":
+		expectLength, proxiedURI, err = kc.Ask(locator)
+	case "GET":
+		reader, expectLength, proxiedURI, err = kc.Get(locator)
 		if reader != nil {
 			defer reader.Close()
 		}
-	} else if req.Method == "HEAD" {
-		blocklen, _, err = kc.AuthorizedAsk(hash, locator.Signature, locator.Timestamp)
+	default:
+		status, err = http.StatusNotImplemented, MethodNotSupported
+		return
 	}
 
-	if blocklen == -1 {
-		log.Printf("%s: %s %s Keep server did not return Content-Length",
-			GetRemoteAddress(req), req.Method, hash)
+	if expectLength == -1 {
+		log.Println("Warning:", GetRemoteAddress(req), req.Method, proxiedURI, "Content-Length not provided")
 	}
 
-	var status = 0
-	switch err {
+	switch respErr := err.(type) {
 	case nil:
 		status = http.StatusOK
-		resp.Header().Set("Content-Length", fmt.Sprint(blocklen))
-		if reader != nil {
-			n, err2 := io.Copy(resp, reader)
-			if blocklen > -1 && n != blocklen {
-				log.Printf("%s: %s %s %v %v mismatched copy size expected Content-Length: %v",
-					GetRemoteAddress(req), req.Method, hash, status, n, blocklen)
-			} else if err2 == nil {
-				log.Printf("%s: %s %s %v %v",
-					GetRemoteAddress(req), req.Method, hash, status, n)
-			} else {
-				log.Printf("%s: %s %s %v %v copy error: %v",
-					GetRemoteAddress(req), req.Method, hash, status, n, err2.Error())
+		resp.Header().Set("Content-Length", fmt.Sprint(expectLength))
+		switch req.Method {
+		case "HEAD":
+			responseLength = 0
+		case "GET":
+			responseLength, err = io.Copy(resp, reader)
+			if err == nil && expectLength > -1 && responseLength != expectLength {
+				err = ContentLengthMismatch
 			}
+		}
+	case keepclient.Error:
+		if respErr == keepclient.BlockNotFound {
+			status = http.StatusNotFound
+		} else if respErr.Temporary() {
+			status = http.StatusBadGateway
 		} else {
-			log.Printf("%s: %s %s %v 0", GetRemoteAddress(req), req.Method, hash, status)
+			status = 422
 		}
-	case keepclient.BlockNotFound:
-		status = http.StatusNotFound
-		http.Error(resp, "Not Found", http.StatusNotFound)
 	default:
-		status = http.StatusBadGateway
-		http.Error(resp, err.Error(), http.StatusBadGateway)
-	}
-
-	if err != nil {
-		log.Printf("%s: %s %s %v error: %v",
-			GetRemoteAddress(req), req.Method, hash, status, err.Error())
+		status = http.StatusInternalServerError
 	}
 }
 
-func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-	SetCorsHeaders(resp)
-
-	kc := *this.KeepClient
+var LengthRequiredError = errors.New(http.StatusText(http.StatusLengthRequired))
+var LengthMismatchError = errors.New("Locator size hint does not match Content-Length header")
 
-	hash := mux.Vars(req)["hash"]
-	hints := mux.Vars(req)["hints"]
+func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) {
+	if err := h.checkLoop(resp, req); err != nil {
+		return
+	}
+	SetCorsHeaders(resp)
+	resp.Header().Set("Via", "HTTP/1.1 "+viaAlias)
 
-	locator := keepclient.MakeLocator2(hash, hints)
+	kc := h.makeKeepClient(req)
 
-	var contentLength int64 = -1
-	if req.Header.Get("Content-Length") != "" {
-		_, err := fmt.Sscanf(req.Header.Get("Content-Length"), "%d", &contentLength)
-		if err != nil {
-			resp.Header().Set("Content-Length", fmt.Sprintf("%d", contentLength))
+	var err error
+	var expectLength int64
+	var status = http.StatusInternalServerError
+	var wroteReplicas int
+	var locatorOut string = "-"
+
+	defer func() {
+		log.Println(GetRemoteAddress(req), req.Method, req.URL.Path, status, expectLength, kc.Want_replicas, wroteReplicas, locatorOut, err)
+		if status != http.StatusOK {
+			http.Error(resp, err.Error(), status)
 		}
+	}()
 
-	}
-
-	log.Printf("%s: %s %s Content-Length %v", GetRemoteAddress(req), req.Method, hash, contentLength)
+	locatorIn := mux.Vars(req)["locator"]
 
-	if contentLength < 0 {
-		http.Error(resp, "Must include Content-Length header", http.StatusLengthRequired)
+	_, err = fmt.Sscanf(req.Header.Get("Content-Length"), "%d", &expectLength)
+	if err != nil || expectLength < 0 {
+		err = LengthRequiredError
+		status = http.StatusLengthRequired
 		return
 	}
 
-	if locator.Size > 0 && int64(locator.Size) != contentLength {
-		http.Error(resp, "Locator size hint does not match Content-Length header", http.StatusBadRequest)
-		return
+	if locatorIn != "" {
+		var loc *keepclient.Locator
+		if loc, err = keepclient.MakeLocator(locatorIn); err != nil {
+			status = http.StatusBadRequest
+			return
+		} else if loc.Size > 0 && int64(loc.Size) != expectLength {
+			err = LengthMismatchError
+			status = http.StatusBadRequest
+			return
+		}
 	}
 
 	var pass bool
 	var tok string
-	if pass, tok = CheckAuthorizationHeader(kc, this.ApiTokenCache, req); !pass {
-		http.Error(resp, "Missing or invalid Authorization header", http.StatusForbidden)
+	if pass, tok = CheckAuthorizationHeader(kc, h.ApiTokenCache, req); !pass {
+		err = BadAuthorizationHeader
+		status = http.StatusForbidden
 		return
 	}
 
@@ -416,63 +484,119 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 	if req.Header.Get("X-Keep-Desired-Replicas") != "" {
 		var r int
 		_, err := fmt.Sscanf(req.Header.Get(keepclient.X_Keep_Desired_Replicas), "%d", &r)
-		if err != nil {
+		if err == nil {
 			kc.Want_replicas = r
 		}
 	}
 
 	// Now try to put the block through
-	var replicas int
-	var put_err error
-	if hash == "" {
+	if locatorIn == "" {
 		if bytes, err := ioutil.ReadAll(req.Body); err != nil {
-			msg := fmt.Sprintf("Error reading request body: %s", err)
-			log.Printf(msg)
-			http.Error(resp, msg, http.StatusInternalServerError)
+			err = errors.New(fmt.Sprintf("Error reading request body: %s", err))
+			status = http.StatusInternalServerError
 			return
 		} else {
-			hash, replicas, put_err = kc.PutB(bytes)
+			locatorOut, wroteReplicas, err = kc.PutB(bytes)
 		}
 	} else {
-		hash, replicas, put_err = kc.PutHR(hash, req.Body, contentLength)
+		locatorOut, wroteReplicas, err = kc.PutHR(locatorIn, req.Body, expectLength)
 	}
 
 	// Tell the client how many successful PUTs we accomplished
-	resp.Header().Set(keepclient.X_Keep_Replicas_Stored, fmt.Sprintf("%d", replicas))
+	resp.Header().Set(keepclient.X_Keep_Replicas_Stored, fmt.Sprintf("%d", wroteReplicas))
 
-	switch put_err {
+	switch err.(type) {
 	case nil:
-		// Default will return http.StatusOK
-		log.Printf("%s: %s %s finished, stored %v replicas (desired %v)", GetRemoteAddress(req), req.Method, hash, replicas, kc.Want_replicas)
-		n, err2 := io.WriteString(resp, hash)
-		if err2 != nil {
-			log.Printf("%s: wrote %v bytes to response body and got error %v", n, err2.Error())
-		}
+		status = http.StatusOK
+		_, err = io.WriteString(resp, locatorOut)
 
 	case keepclient.OversizeBlockError:
 		// Too much data
-		http.Error(resp, fmt.Sprintf("Exceeded maximum blocksize %d", keepclient.BLOCKSIZE), http.StatusRequestEntityTooLarge)
+		status = http.StatusRequestEntityTooLarge
 
 	case keepclient.InsufficientReplicasError:
-		if replicas > 0 {
+		if wroteReplicas > 0 {
 			// At least one write is considered success.  The
 			// client can decide if getting less than the number of
 			// replications it asked for is a fatal error.
-			// Default will return http.StatusOK
-			n, err2 := io.WriteString(resp, hash)
-			if err2 != nil {
-				log.Printf("%s: wrote %v bytes to response body and got error %v", n, err2.Error())
-			}
+			status = http.StatusOK
+			_, err = io.WriteString(resp, locatorOut)
 		} else {
-			http.Error(resp, put_err.Error(), http.StatusServiceUnavailable)
+			status = http.StatusServiceUnavailable
 		}
 
 	default:
-		http.Error(resp, put_err.Error(), http.StatusBadGateway)
+		status = http.StatusBadGateway
 	}
+}
+
+// ServeHTTP implementation for IndexHandler
+// Supports only GET requests for /index/{prefix:[0-9a-f]{0,32}}
+// For each keep server found in LocalRoots:
+//   Invokes GetIndex using keepclient
+//   Expects "complete" response (terminating with blank new line)
+//   Aborts on any errors
+// Concatenates responses from all those keep servers and returns
+func (h *proxyHandler) Index(resp http.ResponseWriter, req *http.Request) {
+	SetCorsHeaders(resp)
+
+	prefix := mux.Vars(req)["prefix"]
+	var err error
+	var status int
+
+	defer func() {
+		if status != http.StatusOK {
+			http.Error(resp, err.Error(), status)
+		}
+	}()
 
-	if put_err != nil {
-		log.Printf("%s: %s %s stored %v replicas (desired %v) got error %v", GetRemoteAddress(req), req.Method, hash, replicas, kc.Want_replicas, put_err.Error())
+	kc := h.makeKeepClient(req)
+	ok, token := CheckAuthorizationHeader(kc, h.ApiTokenCache, req)
+	if !ok {
+		status, err = http.StatusForbidden, BadAuthorizationHeader
+		return
 	}
 
+	// Copy ArvadosClient struct and use the client's API token
+	arvclient := *kc.Arvados
+	arvclient.ApiToken = token
+	kc.Arvados = &arvclient
+
+	// Only GET method is supported
+	if req.Method != "GET" {
+		status, err = http.StatusNotImplemented, MethodNotSupported
+		return
+	}
+
+	// Get index from all LocalRoots and write to resp
+	var reader io.Reader
+	for uuid := range kc.LocalRoots() {
+		reader, err = kc.GetIndex(uuid, prefix)
+		if err != nil {
+			status = http.StatusBadGateway
+			return
+		}
+
+		_, err = io.Copy(resp, reader)
+		if err != nil {
+			status = http.StatusBadGateway
+			return
+		}
+	}
+
+	// Got index from all the keep servers and wrote to resp
+	status = http.StatusOK
+	resp.Write([]byte("\n"))
+}
+
+func (h *proxyHandler) makeKeepClient(req *http.Request) *keepclient.KeepClient {
+	kc := *h.KeepClient
+	kc.HTTPClient = &proxyClient{
+		client: &http.Client{
+			Timeout:   h.timeout,
+			Transport: h.transport,
+		},
+		proto: req.Proto,
+	}
+	return &kc
 }