17119: Merge branch 'master' into 17119-add-filter-groups
[arvados.git] / services / keepproxy / keepproxy.go
index 46664931fd64bbcf40717849cbf8f58abfd43a9b..538a0612275ec029e448b810f45bcdd08fee74bb 100644 (file)
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
-       "encoding/json"
        "errors"
        "flag"
        "fmt"
        "io"
        "io/ioutil"
-       "log"
        "net"
        "net/http"
        "os"
        "os/signal"
        "regexp"
+       "strings"
        "sync"
        "syscall"
        "time"
 
-       "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-       "git.curoverse.com/arvados.git/sdk/go/config"
-       "git.curoverse.com/arvados.git/sdk/go/keepclient"
+       "git.arvados.org/arvados.git/lib/config"
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+       "git.arvados.org/arvados.git/sdk/go/health"
+       "git.arvados.org/arvados.git/sdk/go/httpserver"
+       "git.arvados.org/arvados.git/sdk/go/keepclient"
        "github.com/coreos/go-systemd/daemon"
+       "github.com/ghodss/yaml"
        "github.com/gorilla/mux"
+       log "github.com/sirupsen/logrus"
 )
 
-type Config struct {
-       Client          arvados.Client
-       Listen          string
-       DisableGet      bool
-       DisablePut      bool
-       DefaultReplicas int
-       Timeout         arvados.Duration
-       PIDFile         string
-       Debug           bool
-}
+var version = "dev"
 
-func DefaultConfig() *Config {
-       return &Config{
-               Listen:  ":25107",
-               Timeout: arvados.Duration(15 * time.Second),
-       }
-}
+var (
+       listener net.Listener
+       router   http.Handler
+)
 
-var listener net.Listener
+const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
 
-func main() {
-       cfg := DefaultConfig()
+func configure(logger log.FieldLogger, args []string) (*arvados.Cluster, error) {
+       flags := flag.NewFlagSet(args[0], flag.ExitOnError)
+
+       dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit")
+       getVersion := flags.Bool("version", false, "Print version information and exit.")
 
-       flagset := flag.NewFlagSet("keepproxy", flag.ExitOnError)
-       flagset.Usage = usage
+       loader := config.NewLoader(os.Stdin, logger)
+       loader.SetupFlags(flags)
 
-       const deprecated = " (DEPRECATED -- use config file instead)"
-       flagset.StringVar(&cfg.Listen, "listen", cfg.Listen, "Local port to listen on."+deprecated)
-       flagset.BoolVar(&cfg.DisableGet, "no-get", cfg.DisableGet, "Disable GET operations."+deprecated)
-       flagset.BoolVar(&cfg.DisablePut, "no-put", cfg.DisablePut, "Disable PUT operations."+deprecated)
-       flagset.IntVar(&cfg.DefaultReplicas, "default-replicas", cfg.DefaultReplicas, "Default number of replicas to write if not specified by the client. If 0, use site default."+deprecated)
-       flagset.StringVar(&cfg.PIDFile, "pid", cfg.PIDFile, "Path to write pid file."+deprecated)
-       timeoutSeconds := flagset.Int("timeout", int(time.Duration(cfg.Timeout)/time.Second), "Timeout (in seconds) on requests to internal Keep services."+deprecated)
+       args = loader.MungeLegacyConfigArgs(logger, args[1:], "-legacy-keepproxy-config")
+       flags.Parse(args)
 
-       var cfgPath string
-       const defaultCfgPath = "/etc/arvados/keepproxy/config.json"
-       flagset.StringVar(&cfgPath, "config", defaultCfgPath, "Configuration file `path`")
-       flagset.Parse(os.Args[1:])
+       // Print version information if requested
+       if *getVersion {
+               fmt.Printf("keepproxy %s\n", version)
+               return nil, nil
+       }
 
-       err := config.LoadFile(cfg, cfgPath)
+       cfg, err := loader.Load()
        if err != nil {
-               h := os.Getenv("ARVADOS_API_HOST")
-               t := os.Getenv("ARVADOS_API_TOKEN")
-               if h == "" || t == "" || !os.IsNotExist(err) || cfgPath != defaultCfgPath {
-                       log.Fatal(err)
-               }
-               log.Print("DEPRECATED: No config file found, but ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are set. Please use a config file instead.")
-               cfg.Client.APIHost = h
-               cfg.Client.AuthToken = t
-               if regexp.MustCompile("^(?i:1|yes|true)$").MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE")) {
-                       cfg.Client.Insecure = true
+               return nil, err
+       }
+       cluster, err := cfg.GetCluster("")
+       if err != nil {
+               return nil, err
+       }
+
+       if *dumpConfig {
+               out, err := yaml.Marshal(cfg)
+               if err != nil {
+                       return nil, err
                }
-               if j, err := json.MarshalIndent(cfg, "", "    "); err == nil {
-                       log.Print("Current configuration:\n", string(j))
+               if _, err := os.Stdout.Write(out); err != nil {
+                       return nil, err
                }
-               cfg.Timeout = arvados.Duration(time.Duration(*timeoutSeconds) * time.Second)
+               return nil, nil
+       }
+       return cluster, nil
+}
+
+func main() {
+       logger := log.New()
+       logger.Formatter = &log.JSONFormatter{
+               TimestampFormat: rfc3339NanoFixed,
+       }
+
+       cluster, err := configure(logger, os.Args)
+       if err != nil {
+               log.Fatal(err)
        }
+       if cluster == nil {
+               return
+       }
+
+       log.Printf("keepproxy %s started", version)
+
+       if err := run(logger, cluster); err != nil {
+               log.Fatal(err)
+       }
+
+       log.Println("shutting down")
+}
+
+func run(logger log.FieldLogger, cluster *arvados.Cluster) error {
+       client, err := arvados.NewClientFromConfig(cluster)
+       if err != nil {
+               return err
+       }
+       client.AuthToken = cluster.SystemRootToken
 
-       arv, err := arvadosclient.New(&cfg.Client)
+       arv, err := arvadosclient.New(client)
        if err != nil {
-               log.Fatalf("Error setting up arvados client %s", err.Error())
+               return fmt.Errorf("Error setting up arvados client %v", err)
        }
 
-       if cfg.Debug {
+       // If a config file is available, use the keepstores defined there
+       // instead of the legacy autodiscover mechanism via the API server
+       for k := range cluster.Services.Keepstore.InternalURLs {
+               arv.KeepServiceURIs = append(arv.KeepServiceURIs, strings.TrimRight(k.String(), "/"))
+       }
+
+       if cluster.SystemLogs.LogLevel == "debug" {
                keepclient.DebugPrintf = log.Printf
        }
        kc, err := keepclient.MakeKeepClient(arv)
        if err != nil {
-               log.Fatalf("Error setting up keep client %s", err.Error())
+               return fmt.Errorf("Error setting up keep client %v", err)
        }
+       keepclient.RefreshServiceDiscoveryOnSIGHUP()
 
-       if cfg.PIDFile != "" {
-               f, err := os.Create(cfg.PIDFile)
-               if err != nil {
-                       log.Fatal(err)
-               }
-               defer f.Close()
-               err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
-               if err != nil {
-                       log.Fatalf("flock(%s): %s", cfg.PIDFile, err)
-               }
-               defer os.Remove(cfg.PIDFile)
-               err = f.Truncate(0)
-               if err != nil {
-                       log.Fatalf("truncate(%s): %s", cfg.PIDFile, err)
-               }
-               _, err = fmt.Fprint(f, os.Getpid())
-               if err != nil {
-                       log.Fatalf("write(%s): %s", cfg.PIDFile, err)
-               }
-               err = f.Sync()
-               if err != nil {
-                       log.Fatal("sync(%s): %s", cfg.PIDFile, err)
-               }
+       if cluster.Collections.DefaultReplication > 0 {
+               kc.Want_replicas = cluster.Collections.DefaultReplication
        }
 
-       if cfg.DefaultReplicas > 0 {
-               kc.Want_replicas = cfg.DefaultReplicas
+       var listen arvados.URL
+       for listen = range cluster.Services.Keepproxy.InternalURLs {
+               break
        }
-       kc.Client.Timeout = time.Duration(cfg.Timeout)
-       go kc.RefreshServices(5*time.Minute, 3*time.Second)
 
-       listener, err = net.Listen("tcp", cfg.Listen)
-       if err != nil {
-               log.Fatalf("listen(%s): %s", cfg.Listen, err)
+       var lErr error
+       listener, lErr = net.Listen("tcp", listen.Host)
+       if lErr != nil {
+               return fmt.Errorf("listen(%s): %v", listen.Host, lErr)
        }
-       if _, err := daemon.SdNotify("READY=1"); err != nil {
+
+       if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
                log.Printf("Error notifying init daemon: %v", err)
        }
-       log.Println("Listening at", listener.Addr())
+       log.Println("listening at", listener.Addr())
 
        // Shut down the server gracefully (by closing the listener)
        // if SIGTERM is received.
@@ -148,48 +163,52 @@ func main() {
        signal.Notify(term, syscall.SIGINT)
 
        // Start serving requests.
-       http.Serve(listener, MakeRESTRouter(!cfg.DisableGet, !cfg.DisablePut, kc))
-
-       log.Println("shutting down")
+       router = MakeRESTRouter(kc, time.Duration(keepclient.DefaultProxyRequestTimeout), cluster.ManagementToken)
+       return http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
 }
 
-type ApiTokenCache struct {
+type APITokenCache struct {
        tokens     map[string]int64
        lock       sync.Mutex
        expireTime int64
 }
 
-// Cache the token and set an expire time.  If we already have an expire time
-// on the token, it is not updated.
-func (this *ApiTokenCache) RememberToken(token string) {
-       this.lock.Lock()
-       defer this.lock.Unlock()
+// RememberToken caches the token and set an expire time.  If we already have
+// an expire time on the token, it is not updated.
+func (cache *APITokenCache) RememberToken(token string) {
+       cache.lock.Lock()
+       defer cache.lock.Unlock()
 
        now := time.Now().Unix()
-       if this.tokens[token] == 0 {
-               this.tokens[token] = now + this.expireTime
+       if cache.tokens[token] == 0 {
+               cache.tokens[token] = now + cache.expireTime
        }
 }
 
-// Check if the cached token is known and still believed to be valid.
-func (this *ApiTokenCache) RecallToken(token string) bool {
-       this.lock.Lock()
-       defer this.lock.Unlock()
+// RecallToken checks if the cached token is known and still believed to be
+// valid.
+func (cache *APITokenCache) RecallToken(token string) bool {
+       cache.lock.Lock()
+       defer cache.lock.Unlock()
 
        now := time.Now().Unix()
-       if this.tokens[token] == 0 {
+       if cache.tokens[token] == 0 {
                // Unknown token
                return false
-       } else if now < this.tokens[token] {
+       } else if now < cache.tokens[token] {
                // Token is known and still valid
                return true
        } else {
                // Token is expired
-               this.tokens[token] = 0
+               cache.tokens[token] = 0
                return false
        }
 }
 
+// GetRemoteAddress returns a string with the remote address for the request.
+// If the X-Forwarded-For header is set and has a non-zero length, it returns a
+// string made from a comma separated list of all the remote addresses,
+// starting with the one(s) from the X-Forwarded-For header.
 func GetRemoteAddress(req *http.Request) string {
        if xff := req.Header.Get("X-Forwarded-For"); xff != "" {
                return xff + "," + req.RemoteAddr
@@ -197,91 +216,123 @@ func GetRemoteAddress(req *http.Request) string {
        return req.RemoteAddr
 }
 
-func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *ApiTokenCache, req *http.Request) (pass bool, tok string) {
-       var auth string
-       if auth = req.Header.Get("Authorization"); auth == "" {
+func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *APITokenCache, req *http.Request) (pass bool, tok string) {
+       parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
+       if len(parts) < 2 || !(parts[0] == "OAuth2" || parts[0] == "Bearer") || len(parts[1]) == 0 {
                return false, ""
        }
-
-       _, err := fmt.Sscanf(auth, "OAuth2 %s", &tok)
-       if err != nil {
-               // Scanning error
-               return false, ""
+       tok = parts[1]
+
+       // Tokens are validated differently depending on what kind of
+       // operation is being performed. For example, tokens in
+       // collection-sharing links permit GET requests, but not
+       // PUT requests.
+       var op string
+       if req.Method == "GET" || req.Method == "HEAD" {
+               op = "read"
+       } else {
+               op = "write"
        }
 
-       if cache.RecallToken(tok) {
+       if cache.RecallToken(op + ":" + tok) {
                // Valid in the cache, short circuit
                return true, tok
        }
 
+       var err error
        arv := *kc.Arvados
        arv.ApiToken = tok
-       if err := arv.Call("HEAD", "users", "", "current", nil, nil); err != nil {
+       arv.RequestID = req.Header.Get("X-Request-Id")
+       if op == "read" {
+               err = arv.Call("HEAD", "keep_services", "", "accessible", nil, nil)
+       } else {
+               err = arv.Call("HEAD", "users", "", "current", nil, nil)
+       }
+       if err != nil {
                log.Printf("%s: CheckAuthorizationHeader error: %v", GetRemoteAddress(req), err)
                return false, ""
        }
 
        // Success!  Update cache
-       cache.RememberToken(tok)
+       cache.RememberToken(op + ":" + tok)
 
        return true, tok
 }
 
-type GetBlockHandler struct {
-       *keepclient.KeepClient
-       *ApiTokenCache
-}
+// We need to make a private copy of the default http transport early
+// in initialization, then make copies of our private copy later. It
+// won't be safe to copy http.DefaultTransport itself later, because
+// its private mutexes might have already been used. (Without this,
+// the test suite sometimes panics "concurrent map writes" in
+// net/http.(*Transport).removeIdleConnLocked().)
+var defaultTransport = *(http.DefaultTransport.(*http.Transport))
 
-type PutBlockHandler struct {
+type proxyHandler struct {
+       http.Handler
        *keepclient.KeepClient
-       *ApiTokenCache
+       *APITokenCache
+       timeout   time.Duration
+       transport *http.Transport
 }
 
-type IndexHandler struct {
-       *keepclient.KeepClient
-       *ApiTokenCache
-}
+// MakeRESTRouter returns an http.Handler that passes GET and PUT
+// requests to the appropriate handlers.
+func MakeRESTRouter(kc *keepclient.KeepClient, timeout time.Duration, mgmtToken string) http.Handler {
+       rest := mux.NewRouter()
 
-type InvalidPathHandler struct{}
+       transport := defaultTransport
+       transport.DialContext = (&net.Dialer{
+               Timeout:   keepclient.DefaultConnectTimeout,
+               KeepAlive: keepclient.DefaultKeepAlive,
+               DualStack: true,
+       }).DialContext
+       transport.TLSClientConfig = arvadosclient.MakeTLSConfig(kc.Arvados.ApiInsecure)
+       transport.TLSHandshakeTimeout = keepclient.DefaultTLSHandshakeTimeout
+
+       h := &proxyHandler{
+               Handler:    rest,
+               KeepClient: kc,
+               timeout:    timeout,
+               transport:  &transport,
+               APITokenCache: &APITokenCache{
+                       tokens:     make(map[string]int64),
+                       expireTime: 300,
+               },
+       }
 
-type OptionsHandler struct{}
+       rest.HandleFunc(`/{locator:[0-9a-f]{32}\+.*}`, h.Get).Methods("GET", "HEAD")
+       rest.HandleFunc(`/{locator:[0-9a-f]{32}}`, h.Get).Methods("GET", "HEAD")
 
-// MakeRESTRouter
-//     Returns a mux.Router that passes GET and PUT requests to the
-//     appropriate handlers.
-//
-func MakeRESTRouter(
-       enable_get bool,
-       enable_put bool,
-       kc *keepclient.KeepClient) *mux.Router {
+       // List all blocks
+       rest.HandleFunc(`/index`, h.Index).Methods("GET")
 
-       t := &ApiTokenCache{tokens: make(map[string]int64), expireTime: 300}
+       // List blocks whose hash has the given prefix
+       rest.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, h.Index).Methods("GET")
 
-       rest := mux.NewRouter()
+       rest.HandleFunc(`/{locator:[0-9a-f]{32}\+.*}`, h.Put).Methods("PUT")
+       rest.HandleFunc(`/{locator:[0-9a-f]{32}}`, h.Put).Methods("PUT")
+       rest.HandleFunc(`/`, h.Put).Methods("POST")
+       rest.HandleFunc(`/{any}`, h.Options).Methods("OPTIONS")
+       rest.HandleFunc(`/`, h.Options).Methods("OPTIONS")
 
-       if enable_get {
-               rest.Handle(`/{locator:[0-9a-f]{32}\+.*}`,
-                       GetBlockHandler{kc, t}).Methods("GET", "HEAD")
-               rest.Handle(`/{locator:[0-9a-f]{32}}`, GetBlockHandler{kc, t}).Methods("GET", "HEAD")
+       rest.Handle("/_health/{check}", &health.Handler{
+               Token:  mgmtToken,
+               Prefix: "/_health/",
+       }).Methods("GET")
 
-               // List all blocks
-               rest.Handle(`/index`, IndexHandler{kc, t}).Methods("GET")
+       rest.NotFoundHandler = InvalidPathHandler{}
+       return h
+}
 
-               // List blocks whose hash has the given prefix
-               rest.Handle(`/index/{prefix:[0-9a-f]{0,32}}`, IndexHandler{kc, t}).Methods("GET")
-       }
+var errLoopDetected = errors.New("loop detected")
 
-       if enable_put {
-               rest.Handle(`/{locator:[0-9a-f]{32}\+.*}`, PutBlockHandler{kc, t}).Methods("PUT")
-               rest.Handle(`/{locator:[0-9a-f]{32}}`, PutBlockHandler{kc, t}).Methods("PUT")
-               rest.Handle(`/`, PutBlockHandler{kc, t}).Methods("POST")
-               rest.Handle(`/{any}`, OptionsHandler{}).Methods("OPTIONS")
-               rest.Handle(`/`, OptionsHandler{}).Methods("OPTIONS")
+func (*proxyHandler) checkLoop(resp http.ResponseWriter, req *http.Request) error {
+       if via := req.Header.Get("Via"); strings.Index(via, " "+viaAlias) >= 0 {
+               log.Printf("proxy loop detected (request has Via: %q): perhaps keepproxy is misidentified by gateway config as an external client, or its keep_services record does not have service_type=proxy?", via)
+               http.Error(resp, errLoopDetected.Error(), http.StatusInternalServerError)
+               return errLoopDetected
        }
-
-       rest.NotFoundHandler = InvalidPathHandler{}
-
-       return rest
+       return nil
 }
 
 func SetCorsHeaders(resp http.ResponseWriter) {
@@ -291,24 +342,30 @@ func SetCorsHeaders(resp http.ResponseWriter) {
        resp.Header().Set("Access-Control-Max-Age", "86486400")
 }
 
-func (this InvalidPathHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+type InvalidPathHandler struct{}
+
+func (InvalidPathHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
        log.Printf("%s: %s %s unroutable", GetRemoteAddress(req), req.Method, req.URL.Path)
        http.Error(resp, "Bad request", http.StatusBadRequest)
 }
 
-func (this OptionsHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h *proxyHandler) Options(resp http.ResponseWriter, req *http.Request) {
        log.Printf("%s: %s %s", GetRemoteAddress(req), req.Method, req.URL.Path)
        SetCorsHeaders(resp)
 }
 
-var BadAuthorizationHeader = errors.New("Missing or invalid Authorization header")
-var ContentLengthMismatch = errors.New("Actual length != expected content length")
-var MethodNotSupported = errors.New("Method not supported")
+var errBadAuthorizationHeader = errors.New("Missing or invalid Authorization header")
+var errContentLengthMismatch = errors.New("Actual length != expected content length")
+var errMethodNotSupported = errors.New("Method not supported")
 
 var removeHint, _ = regexp.Compile("\\+K@[a-z0-9]{5}(\\+|$)")
 
-func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h *proxyHandler) Get(resp http.ResponseWriter, req *http.Request) {
+       if err := h.checkLoop(resp, req); err != nil {
+               return
+       }
        SetCorsHeaders(resp)
+       resp.Header().Set("Via", req.Proto+" "+viaAlias)
 
        locator := mux.Vars(req)["locator"]
        var err error
@@ -323,12 +380,12 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
                }
        }()
 
-       kc := *this.KeepClient
+       kc := h.makeKeepClient(req)
 
        var pass bool
        var tok string
-       if pass, tok = CheckAuthorizationHeader(&kc, this.ApiTokenCache, req); !pass {
-               status, err = http.StatusForbidden, BadAuthorizationHeader
+       if pass, tok = CheckAuthorizationHeader(kc, h.APITokenCache, req); !pass {
+               status, err = http.StatusForbidden, errBadAuthorizationHeader
                return
        }
 
@@ -350,7 +407,7 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
                        defer reader.Close()
                }
        default:
-               status, err = http.StatusNotImplemented, MethodNotSupported
+               status, err = http.StatusNotImplemented, errMethodNotSupported
                return
        }
 
@@ -368,7 +425,7 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
                case "GET":
                        responseLength, err = io.Copy(resp, reader)
                        if err == nil && expectLength > -1 && responseLength != expectLength {
-                               err = ContentLengthMismatch
+                               err = errContentLengthMismatch
                        }
                }
        case keepclient.Error:
@@ -384,13 +441,18 @@ func (this GetBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
        }
 }
 
-var LengthRequiredError = errors.New(http.StatusText(http.StatusLengthRequired))
-var LengthMismatchError = errors.New("Locator size hint does not match Content-Length header")
+var errLengthRequired = errors.New(http.StatusText(http.StatusLengthRequired))
+var errLengthMismatch = errors.New("Locator size hint does not match Content-Length header")
 
-func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) {
+       if err := h.checkLoop(resp, req); err != nil {
+               return
+       }
        SetCorsHeaders(resp)
+       resp.Header().Set("Via", "HTTP/1.1 "+viaAlias)
+
+       kc := h.makeKeepClient(req)
 
-       kc := *this.KeepClient
        var err error
        var expectLength int64
        var status = http.StatusInternalServerError
@@ -406,9 +468,18 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 
        locatorIn := mux.Vars(req)["locator"]
 
+       // Check if the client specified storage classes
+       if req.Header.Get("X-Keep-Storage-Classes") != "" {
+               var scl []string
+               for _, sc := range strings.Split(req.Header.Get("X-Keep-Storage-Classes"), ",") {
+                       scl = append(scl, strings.Trim(sc, " "))
+               }
+               kc.StorageClasses = scl
+       }
+
        _, err = fmt.Sscanf(req.Header.Get("Content-Length"), "%d", &expectLength)
        if err != nil || expectLength < 0 {
-               err = LengthRequiredError
+               err = errLengthRequired
                status = http.StatusLengthRequired
                return
        }
@@ -419,7 +490,7 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
                        status = http.StatusBadRequest
                        return
                } else if loc.Size > 0 && int64(loc.Size) != expectLength {
-                       err = LengthMismatchError
+                       err = errLengthMismatch
                        status = http.StatusBadRequest
                        return
                }
@@ -427,8 +498,8 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 
        var pass bool
        var tok string
-       if pass, tok = CheckAuthorizationHeader(&kc, this.ApiTokenCache, req); !pass {
-               err = BadAuthorizationHeader
+       if pass, tok = CheckAuthorizationHeader(kc, h.APITokenCache, req); !pass {
+               err = errBadAuthorizationHeader
                status = http.StatusForbidden
                return
        }
@@ -441,7 +512,7 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
        // Check if the client specified the number of replicas
        if req.Header.Get("X-Keep-Desired-Replicas") != "" {
                var r int
-               _, err := fmt.Sscanf(req.Header.Get(keepclient.X_Keep_Desired_Replicas), "%d", &r)
+               _, err := fmt.Sscanf(req.Header.Get(keepclient.XKeepDesiredReplicas), "%d", &r)
                if err == nil {
                        kc.Want_replicas = r
                }
@@ -449,21 +520,21 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 
        // Now try to put the block through
        if locatorIn == "" {
-               if bytes, err := ioutil.ReadAll(req.Body); err != nil {
-                       err = errors.New(fmt.Sprintf("Error reading request body: %s", err))
+               bytes, err2 := ioutil.ReadAll(req.Body)
+               if err2 != nil {
+                       err = fmt.Errorf("Error reading request body: %s", err2)
                        status = http.StatusInternalServerError
                        return
-               } else {
-                       locatorOut, wroteReplicas, err = kc.PutB(bytes)
                }
+               locatorOut, wroteReplicas, err = kc.PutB(bytes)
        } else {
                locatorOut, wroteReplicas, err = kc.PutHR(locatorIn, req.Body, expectLength)
        }
 
        // Tell the client how many successful PUTs we accomplished
-       resp.Header().Set(keepclient.X_Keep_Replicas_Stored, fmt.Sprintf("%d", wroteReplicas))
+       resp.Header().Set(keepclient.XKeepReplicasStored, fmt.Sprintf("%d", wroteReplicas))
 
-       switch err {
+       switch err.(type) {
        case nil:
                status = http.StatusOK
                _, err = io.WriteString(resp, locatorOut)
@@ -495,7 +566,7 @@ func (this PutBlockHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 //   Expects "complete" response (terminating with blank new line)
 //   Aborts on any errors
 // Concatenates responses from all those keep servers and returns
-func (handler IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h *proxyHandler) Index(resp http.ResponseWriter, req *http.Request) {
        SetCorsHeaders(resp)
 
        prefix := mux.Vars(req)["prefix"]
@@ -508,11 +579,10 @@ func (handler IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
                }
        }()
 
-       kc := *handler.KeepClient
-
-       ok, token := CheckAuthorizationHeader(&kc, handler.ApiTokenCache, req)
+       kc := h.makeKeepClient(req)
+       ok, token := CheckAuthorizationHeader(kc, h.APITokenCache, req)
        if !ok {
-               status, err = http.StatusForbidden, BadAuthorizationHeader
+               status, err = http.StatusForbidden, errBadAuthorizationHeader
                return
        }
 
@@ -523,7 +593,7 @@ func (handler IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
 
        // Only GET method is supported
        if req.Method != "GET" {
-               status, err = http.StatusNotImplemented, MethodNotSupported
+               status, err = http.StatusNotImplemented, errMethodNotSupported
                return
        }
 
@@ -547,3 +617,16 @@ func (handler IndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Reques
        status = http.StatusOK
        resp.Write([]byte("\n"))
 }
+
+func (h *proxyHandler) makeKeepClient(req *http.Request) *keepclient.KeepClient {
+       kc := *h.KeepClient
+       kc.RequestID = req.Header.Get("X-Request-Id")
+       kc.HTTPClient = &proxyClient{
+               client: &http.Client{
+                       Timeout:   h.timeout,
+                       Transport: h.transport,
+               },
+               proto: req.Proto,
+       }
+       return &kc
+}