"Check health of all Arvados cluster services"
package_go_binary services/keep-balance keep-balance "$FORMAT" "$ARCH" \
"Rebalance and garbage-collect data blocks stored in Arvados Keep"
-package_go_binary services/keepproxy keepproxy "$FORMAT" "$ARCH" \
+package_go_binary cmd/arvados-server keepproxy "$FORMAT" "$ARCH" \
"Make a Keep cluster accessible to clients that are not on the LAN"
package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \
"Keep storage daemon, accessible to clients on the LAN"
"git.arvados.org/arvados.git/lib/install"
"git.arvados.org/arvados.git/lib/lsf"
"git.arvados.org/arvados.git/lib/recovercollection"
+ "git.arvados.org/arvados.git/services/keepproxy"
"git.arvados.org/arvados.git/services/keepstore"
"git.arvados.org/arvados.git/services/ws"
)
"dispatch-lsf": lsf.DispatchCommand,
"install": install.Command,
"init": install.InitCommand,
+ "keepproxy": keepproxy.Command,
"keepstore": keepstore.Command,
"recover-collection": recovercollection.Command,
"ws": ws.Command,
runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}},
runGoProgram{src: "services/arv-git-httpd", svc: super.cluster.Services.GitHTTP},
runGoProgram{src: "services/health", svc: super.cluster.Services.Health},
- runGoProgram{src: "services/keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
+ runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
runGoProgram{src: "services/keep-web", svc: super.cluster.Services.WebDAV},
runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepproxy
import (
"context"
"errors"
- "flag"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
- "os"
- "os/signal"
"regexp"
"strings"
- "syscall"
"time"
- "git.arvados.org/arvados.git/lib/cmd"
- "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
- "github.com/coreos/go-systemd/daemon"
- "github.com/ghodss/yaml"
"github.com/gorilla/mux"
lru "github.com/hashicorp/golang-lru"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
)
-var version = "dev"
-
-var (
- listener net.Listener
- router http.Handler
-)
-
const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-func configure(args []string) (*arvados.Cluster, logrus.FieldLogger, error) {
- prog := args[0]
- flags := flag.NewFlagSet(prog, flag.ContinueOnError)
-
- dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit")
- getVersion := flags.Bool("version", false, "Print version information and exit.")
-
- initLogger := logrus.New()
- initLogger.Formatter = &logrus.JSONFormatter{
- TimestampFormat: rfc3339NanoFixed,
- }
- var logger logrus.FieldLogger = initLogger
-
- loader := config.NewLoader(os.Stdin, logger)
- loader.SetupFlags(flags)
- args = loader.MungeLegacyConfigArgs(logger, args[1:], "-legacy-keepproxy-config")
-
- if ok, code := cmd.ParseFlags(flags, prog, args, "", os.Stderr); !ok {
- os.Exit(code)
- } else if *getVersion {
- fmt.Printf("keepproxy %s\n", version)
- return nil, logger, nil
- }
-
- cfg, err := loader.Load()
- if err != nil {
- return nil, logger, err
- }
- cluster, err := cfg.GetCluster("")
- if err != nil {
- return nil, logger, err
- }
-
- logger = ctxlog.New(os.Stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
- "ClusterID": cluster.ClusterID,
- "PID": os.Getpid(),
- })
-
- if *dumpConfig {
- out, err := yaml.Marshal(cfg)
- if err != nil {
- return nil, logger, err
- }
- if _, err := os.Stdout.Write(out); err != nil {
- return nil, logger, err
- }
- return nil, logger, nil
- }
+var Command = service.Command(arvados.ServiceNameKeepproxy, newHandlerOrErrorHandler)
- return cluster, logger, nil
-}
-
-func main() {
- cluster, logger, err := configure(os.Args)
- if err != nil {
- logger.Fatal(err)
- }
- if cluster == nil {
- return
- }
-
- logger.Printf("keepproxy %s started", version)
-
- if err := run(logger, cluster); err != nil {
- logger.Fatal(err)
- }
-
- logger.Println("shutting down")
-}
-
-func run(logger logrus.FieldLogger, cluster *arvados.Cluster) error {
+func newHandlerOrErrorHandler(ctx context.Context, cluster *arvados.Cluster, token string, reg *prometheus.Registry) service.Handler {
client, err := arvados.NewClientFromConfig(cluster)
if err != nil {
- return err
+ return service.ErrorHandler(ctx, cluster, fmt.Errorf("Error setting up arvados client: %w", err))
}
- client.AuthToken = cluster.SystemRootToken
-
arv, err := arvadosclient.New(client)
if err != nil {
- return fmt.Errorf("Error setting up arvados client %v", err)
- }
-
- // If a config file is available, use the keepstores defined there
- // instead of the legacy autodiscover mechanism via the API server
- for k := range cluster.Services.Keepstore.InternalURLs {
- arv.KeepServiceURIs = append(arv.KeepServiceURIs, strings.TrimRight(k.String(), "/"))
- }
-
- if cluster.SystemLogs.LogLevel == "debug" {
- keepclient.DebugPrintf = logger.Printf
+ return service.ErrorHandler(ctx, cluster, fmt.Errorf("Error setting up arvados client: %w", err))
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
- return fmt.Errorf("Error setting up keep client %v", err)
+ return service.ErrorHandler(ctx, cluster, fmt.Errorf("Error setting up keep client: %w", err))
}
keepclient.RefreshServiceDiscoveryOnSIGHUP()
-
- if cluster.Collections.DefaultReplication > 0 {
- kc.Want_replicas = cluster.Collections.DefaultReplication
- }
-
- var listen arvados.URL
- for listen = range cluster.Services.Keepproxy.InternalURLs {
- break
- }
-
- var lErr error
- listener, lErr = net.Listen("tcp", listen.Host)
- if lErr != nil {
- return fmt.Errorf("listen(%s): %v", listen.Host, lErr)
- }
-
- if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- logger.Printf("Error notifying init daemon: %v", err)
- }
- logger.Println("listening at", listener.Addr())
-
- // Shut down the server gracefully (by closing the listener)
- // if SIGTERM is received.
- term := make(chan os.Signal, 1)
- go func(sig <-chan os.Signal) {
- s := <-sig
- logger.Println("caught signal:", s)
- listener.Close()
- }(term)
- signal.Notify(term, syscall.SIGTERM)
- signal.Notify(term, syscall.SIGINT)
-
- // Start serving requests.
- router, err = MakeRESTRouter(kc, time.Duration(keepclient.DefaultProxyRequestTimeout), cluster, logger)
+ router, err := newHandler(ctx, kc, time.Duration(keepclient.DefaultProxyRequestTimeout), cluster)
if err != nil {
- return err
- }
- server := http.Server{
- Handler: httpserver.AddRequestIDs(httpserver.LogRequests(router)),
- BaseContext: func(net.Listener) context.Context {
- return ctxlog.Context(context.Background(), logger)
- },
+ return service.ErrorHandler(ctx, cluster, err)
}
- return server.Serve(listener)
+ return router
}
-type TokenCacheEntry struct {
+type tokenCacheEntry struct {
expire int64
user *arvados.User
}
-type APITokenCache struct {
+type apiTokenCache struct {
tokens *lru.TwoQueueCache
expireTime int64
}
// RememberToken caches the token and set an expire time. If the
// token is already in the cache, it is not updated.
-func (cache *APITokenCache) RememberToken(token string, user *arvados.User) {
+func (cache *apiTokenCache) RememberToken(token string, user *arvados.User) {
now := time.Now().Unix()
_, ok := cache.tokens.Get(token)
if !ok {
- cache.tokens.Add(token, TokenCacheEntry{
+ cache.tokens.Add(token, tokenCacheEntry{
expire: now + cache.expireTime,
user: user,
})
// RecallToken checks if the cached token is known and still believed to be
// valid.
-func (cache *APITokenCache) RecallToken(token string) (bool, *arvados.User) {
+func (cache *apiTokenCache) RecallToken(token string) (bool, *arvados.User) {
val, ok := cache.tokens.Get(token)
if !ok {
return false, nil
}
- cacheEntry := val.(TokenCacheEntry)
+ cacheEntry := val.(tokenCacheEntry)
now := time.Now().Unix()
if now < cacheEntry.expire {
// Token is known and still valid
}
}
-// GetRemoteAddress returns a string with the remote address for the request.
-// If the X-Forwarded-For header is set and has a non-zero length, it returns a
-// string made from a comma separated list of all the remote addresses,
-// starting with the one(s) from the X-Forwarded-For header.
-func GetRemoteAddress(req *http.Request) string {
- if xff := req.Header.Get("X-Forwarded-For"); xff != "" {
- return xff + "," + req.RemoteAddr
- }
- return req.RemoteAddr
+func (h *proxyHandler) Done() <-chan struct{} {
+ return nil
+}
+
+func (h *proxyHandler) CheckHealth() error {
+ return nil
}
-func (h *proxyHandler) CheckAuthorizationHeader(req *http.Request) (pass bool, tok string, user *arvados.User) {
+func (h *proxyHandler) checkAuthorizationHeader(req *http.Request) (pass bool, tok string, user *arvados.User) {
parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
if len(parts) < 2 || !(parts[0] == "OAuth2" || parts[0] == "Bearer") || len(parts[1]) == 0 {
return false, "", nil
op = "write"
}
- if ok, user := h.APITokenCache.RecallToken(op + ":" + tok); ok {
+ if ok, user := h.apiTokenCache.RecallToken(op + ":" + tok); ok {
// Valid in the cache, short circuit
return true, tok, user
}
}
}
if err != nil {
- ctxlog.FromContext(req.Context()).Printf("%s: CheckAuthorizationHeader error: %v", GetRemoteAddress(req), err)
+ ctxlog.FromContext(req.Context()).WithError(err).Info("checkAuthorizationHeader error")
return false, "", nil
}
}
// Success! Update cache
- h.APITokenCache.RememberToken(op+":"+tok, user)
+ h.apiTokenCache.RememberToken(op+":"+tok, user)
return true, tok, user
}
type proxyHandler struct {
http.Handler
*keepclient.KeepClient
- *APITokenCache
+ *apiTokenCache
timeout time.Duration
transport *http.Transport
- logger logrus.FieldLogger
cluster *arvados.Cluster
}
-// MakeRESTRouter returns an http.Handler that passes GET and PUT
-// requests to the appropriate handlers.
-func MakeRESTRouter(kc *keepclient.KeepClient, timeout time.Duration, cluster *arvados.Cluster, logger logrus.FieldLogger) (http.Handler, error) {
+func newHandler(ctx context.Context, kc *keepclient.KeepClient, timeout time.Duration, cluster *arvados.Cluster) (service.Handler, error) {
rest := mux.NewRouter()
transport := defaultTransport
KeepClient: kc,
timeout: timeout,
transport: &transport,
- APITokenCache: &APITokenCache{
+ apiTokenCache: &apiTokenCache{
tokens: cacheQ,
expireTime: 300,
},
- logger: logger,
cluster: cluster,
}
Prefix: "/_health/",
}).Methods("GET")
- rest.NotFoundHandler = InvalidPathHandler{}
+ rest.NotFoundHandler = invalidPathHandler{}
return h, nil
}
func (h *proxyHandler) checkLoop(resp http.ResponseWriter, req *http.Request) error {
if via := req.Header.Get("Via"); strings.Index(via, " "+viaAlias) >= 0 {
- h.logger.Printf("proxy loop detected (request has Via: %q): perhaps keepproxy is misidentified by gateway config as an external client, or its keep_services record does not have service_type=proxy?", via)
+ ctxlog.FromContext(req.Context()).Printf("proxy loop detected (request has Via: %q): perhaps keepproxy is misidentified by gateway config as an external client, or its keep_services record does not have service_type=proxy?", via)
http.Error(resp, errLoopDetected.Error(), http.StatusInternalServerError)
return errLoopDetected
}
return nil
}
-func SetCorsHeaders(resp http.ResponseWriter) {
+func setCORSHeaders(resp http.ResponseWriter) {
resp.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, POST, PUT, OPTIONS")
resp.Header().Set("Access-Control-Allow-Origin", "*")
resp.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
resp.Header().Set("Access-Control-Max-Age", "86486400")
}
-type InvalidPathHandler struct{}
+type invalidPathHandler struct{}
-func (InvalidPathHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- ctxlog.FromContext(req.Context()).Printf("%s: %s %s unroutable", GetRemoteAddress(req), req.Method, req.URL.Path)
+func (invalidPathHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
http.Error(resp, "Bad request", http.StatusBadRequest)
}
func (h *proxyHandler) Options(resp http.ResponseWriter, req *http.Request) {
- ctxlog.FromContext(req.Context()).Printf("%s: %s %s", GetRemoteAddress(req), req.Method, req.URL.Path)
- SetCorsHeaders(resp)
+ setCORSHeaders(resp)
}
var errBadAuthorizationHeader = errors.New("Missing or invalid Authorization header, or method not allowed")
if err := h.checkLoop(resp, req); err != nil {
return
}
- SetCorsHeaders(resp)
+ setCORSHeaders(resp)
resp.Header().Set("Via", req.Proto+" "+viaAlias)
locator := mux.Vars(req)["locator"]
var expectLength, responseLength int64
var proxiedURI = "-"
+ logger := ctxlog.FromContext(req.Context())
defer func() {
- h.logger.Println(GetRemoteAddress(req), req.Method, req.URL.Path, status, expectLength, responseLength, proxiedURI, err)
+ httpserver.SetResponseLogFields(req.Context(), logrus.Fields{
+ "expectLength": expectLength,
+ "responseLength": responseLength,
+ "proxiedURI": proxiedURI,
+ "err": err,
+ })
if status != http.StatusOK {
http.Error(resp, err.Error(), status)
}
var pass bool
var tok string
var user *arvados.User
- if pass, tok, user = h.CheckAuthorizationHeader(req); !pass {
+ if pass, tok, user = h.checkAuthorizationHeader(req); !pass {
status, err = http.StatusForbidden, errBadAuthorizationHeader
return
}
+ httpserver.SetResponseLogFields(req.Context(), logrus.Fields{
+ "userUUID": user.UUID,
+ "userFullName": user.FullName,
+ })
// Copy ArvadosClient struct and use the client's API token
arvclient := *kc.Arvados
locator = removeHint.ReplaceAllString(locator, "$1")
- if locator != "" {
- parts := strings.SplitN(locator, "+", 3)
- if len(parts) >= 2 {
- logger := h.logger
- if user != nil {
- logger = logger.WithField("user_uuid", user.UUID).
- WithField("user_full_name", user.FullName)
- }
- logger.WithField("locator", fmt.Sprintf("%s+%s", parts[0], parts[1])).Infof("Block download")
- }
- }
-
switch req.Method {
case "HEAD":
expectLength, proxiedURI, err = kc.Ask(locator)
}
if expectLength == -1 {
- h.logger.Println("Warning:", GetRemoteAddress(req), req.Method, proxiedURI, "Content-Length not provided")
+ logger.Warn("Content-Length not provided")
}
switch respErr := err.(type) {
if err := h.checkLoop(resp, req); err != nil {
return
}
- SetCorsHeaders(resp)
+ setCORSHeaders(resp)
resp.Header().Set("Via", "HTTP/1.1 "+viaAlias)
kc := h.makeKeepClient(req)
var locatorOut string = "-"
defer func() {
- h.logger.Println(GetRemoteAddress(req), req.Method, req.URL.Path, status, expectLength, kc.Want_replicas, wroteReplicas, locatorOut, err)
+ httpserver.SetResponseLogFields(req.Context(), logrus.Fields{
+ "expectLength": expectLength,
+ "wantReplicas": kc.Want_replicas,
+ "wroteReplicas": wroteReplicas,
+ "locator": strings.SplitN(locatorOut, "+A", 2)[0],
+ "err": err,
+ })
if status != http.StatusOK {
http.Error(resp, err.Error(), status)
}
var pass bool
var tok string
var user *arvados.User
- if pass, tok, user = h.CheckAuthorizationHeader(req); !pass {
+ if pass, tok, user = h.checkAuthorizationHeader(req); !pass {
err = errBadAuthorizationHeader
status = http.StatusForbidden
return
}
+ httpserver.SetResponseLogFields(req.Context(), logrus.Fields{
+ "userUUID": user.UUID,
+ "userFullName": user.FullName,
+ })
// Copy ArvadosClient struct and use the client's API token
arvclient := *kc.Arvados
locatorOut, wroteReplicas, err = kc.PutHR(locatorIn, req.Body, expectLength)
}
- if locatorOut != "" {
- parts := strings.SplitN(locatorOut, "+", 3)
- if len(parts) >= 2 {
- logger := h.logger
- if user != nil {
- logger = logger.WithField("user_uuid", user.UUID).
- WithField("user_full_name", user.FullName)
- }
- logger.WithField("locator", fmt.Sprintf("%s+%s", parts[0], parts[1])).Infof("Block upload")
- }
- }
-
// Tell the client how many successful PUTs we accomplished
resp.Header().Set(keepclient.XKeepReplicasStored, fmt.Sprintf("%d", wroteReplicas))
// Aborts on any errors
// Concatenates responses from all those keep servers and returns
func (h *proxyHandler) Index(resp http.ResponseWriter, req *http.Request) {
- SetCorsHeaders(resp)
+ setCORSHeaders(resp)
prefix := mux.Vars(req)["prefix"]
var err error
}()
kc := h.makeKeepClient(req)
- ok, token, _ := h.CheckAuthorizationHeader(req)
+ ok, token, _ := h.checkAuthorizationHeader(req)
if !ok {
status, err = http.StatusForbidden, errBadAuthorizationHeader
return
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-[Unit]
-Description=Arvados Keep Proxy
-Documentation=https://doc.arvados.org/
-After=network.target
-
-# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
-StartLimitIntervalSec=0
-
-[Service]
-Type=notify
-ExecStart=/usr/bin/keepproxy
-# Set a reasonable default for the open file limit
-LimitNOFILE=65536
-Restart=always
-RestartSec=1
-
-# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
-StartLimitInterval=0
-
-[Install]
-WantedBy=multi-user.target
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepproxy
import (
"bytes"
+ "context"
"crypto/md5"
"fmt"
"io/ioutil"
"math/rand"
+ "net"
"net/http"
"net/http/httptest"
"strings"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
log "github.com/sirupsen/logrus"
var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz"
-// Wait (up to 1 second) for keepproxy to listen on a port. This
-// avoids a race condition where we hit a "connection refused" error
-// because we start testing the proxy too soon.
-func waitForListener() {
- const (
- ms = 5
- )
- for i := 0; listener == nil && i < 10000; i += ms {
- time.Sleep(ms * time.Millisecond)
- }
- if listener == nil {
- panic("Timed out waiting for listener to start")
- }
-}
-
-func closeListener() {
- if listener != nil {
- listener.Close()
- }
-}
-
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
arvadostest.StartKeep(2, false)
}
arvadostest.ResetEnv()
}
-func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*keepclient.KeepClient, *bytes.Buffer) {
+type testServer struct {
+ *httpserver.Server
+ proxyHandler *proxyHandler
+}
+
+func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool, kp *arvados.UploadDownloadRolePermissions) (*testServer, *keepclient.KeepClient, *bytes.Buffer) {
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
c.Assert(err, Equals, nil)
cluster, err := cfg.GetCluster("")
cluster.Collections.KeepproxyPermission = *kp
}
- listener = nil
logbuf := &bytes.Buffer{}
logger := log.New()
logger.Out = logbuf
- go func() {
- run(logger, cluster)
- defer closeListener()
- }()
- waitForListener()
+ ctx := ctxlog.Context(context.Background(), logger)
+
+ handler := newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, nil).(*proxyHandler)
+ srv := &testServer{
+ Server: &httpserver.Server{
+ Server: http.Server{
+ BaseContext: func(net.Listener) context.Context { return ctx },
+ Handler: httpserver.AddRequestIDs(
+ httpserver.LogRequests(handler)),
+ },
+ Addr: ":",
+ },
+ proxyHandler: handler,
+ }
+ err = srv.Start()
+ c.Assert(err, IsNil)
client := arvados.NewClientFromEnv()
arv, err := arvadosclient.New(client)
- c.Assert(err, Equals, nil)
+ c.Assert(err, IsNil)
if bogusClientToken {
arv.ApiToken = "bogus-token"
}
kc := keepclient.New(arv)
sr := map[string]string{
- TestProxyUUID: "http://" + listener.Addr().String(),
+ TestProxyUUID: "http://" + srv.Addr,
}
kc.SetServiceRoots(sr, sr, sr)
kc.Arvados.External = true
-
- return kc, logbuf
+ return srv, kc, logbuf
}
func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
- runProxy(c, false, false, nil)
- defer closeListener()
+ srv, _, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
req, err := http.NewRequest("POST",
- "http://"+listener.Addr().String()+"/",
+ "http://"+srv.Addr+"/",
strings.NewReader("TestViaHeader"))
c.Assert(err, Equals, nil)
req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
resp.Body.Close()
req, err = http.NewRequest("GET",
- "http://"+listener.Addr().String()+"/"+string(locator),
+ "http://"+srv.Addr+"/"+string(locator),
nil)
c.Assert(err, Equals, nil)
resp, err = (&http.Client{}).Do(req)
}
func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
sr := map[string]string{
- TestProxyUUID: "http://" + listener.Addr().String(),
+ TestProxyUUID: "http://" + srv.Addr,
}
- router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
+ srv.proxyHandler.KeepClient.SetServiceRoots(sr, sr, sr)
content := []byte("TestLoopDetection")
_, _, err := kc.PutB(content)
}
func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
// Set up fake keepstore to record request headers
var hdr http.Header
sr := map[string]string{
TestProxyUUID: ts.URL,
}
- router.(*proxyHandler).KeepClient.SetServiceRoots(sr, sr, sr)
+ srv.proxyHandler.KeepClient.SetServiceRoots(sr, sr, sr)
// Set up client to ask for storage classes to keepproxy
kc.StorageClasses = []string{"secure"}
}
func (s *ServerRequiredSuite) TestStorageClassesConfirmedHeader(c *C) {
- runProxy(c, false, false, nil)
- defer closeListener()
+ srv, _, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
content := []byte("foo")
hash := fmt.Sprintf("%x", md5.Sum(content))
client := &http.Client{}
req, err := http.NewRequest("PUT",
- fmt.Sprintf("http://%s/%s", listener.Addr().String(), hash),
+ fmt.Sprintf("http://%s/%s", srv.Addr, hash),
bytes.NewReader(content))
c.Assert(err, IsNil)
req.Header.Set("X-Keep-Storage-Classes", "default")
}
func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
content := []byte("TestDesiredReplicas")
hash := fmt.Sprintf("%x", md5.Sum(content))
}
func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
content := []byte("TestPutWrongContentLength")
hash := fmt.Sprintf("%x", md5.Sum(content))
// fixes the invalid Content-Length header. In order to test
// our server behavior, we have to call the handler directly
// using an httptest.ResponseRecorder.
- rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{}, log.New())
+ rtr, err := newHandler(context.Background(), kc, 10*time.Second, &arvados.Cluster{})
c.Assert(err, check.IsNil)
type testcase struct {
{"abcdef", http.StatusLengthRequired},
} {
req, err := http.NewRequest("PUT",
- fmt.Sprintf("http://%s/%s+%d", listener.Addr().String(), hash, len(content)),
+ fmt.Sprintf("http://%s/%s+%d", srv.Addr, hash, len(content)),
bytes.NewReader(content))
c.Assert(err, IsNil)
req.Header.Set("Content-Length", t.sendLength)
}
func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
- router.(*proxyHandler).timeout = time.Nanosecond
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
+ srv.proxyHandler.timeout = time.Nanosecond
buf := make([]byte, 1<<20)
rand.Read(buf)
}
func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
- kc, logbuf := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, logbuf := runProxy(c, false, false, nil)
+ defer srv.Close()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
var hash2 string
c.Check(err, Equals, nil)
c.Log("Finished PutB (expected success)")
- c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
+ c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload".* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* user_full_name="TestCase Administrator".* user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
logbuf.Reset()
}
c.Assert(err, Equals, nil)
c.Check(blocklen, Equals, int64(3))
c.Log("Finished Ask (expected success)")
- c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
+ c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download".* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* user_full_name="TestCase Administrator".* user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
logbuf.Reset()
}
c.Check(all, DeepEquals, []byte("foo"))
c.Check(blocklen, Equals, int64(3))
c.Log("Finished Get (expected success)")
- c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
+ c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download".* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* user_full_name="TestCase Administrator".* user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
logbuf.Reset()
}
}
func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
- kc, _ := runProxy(c, true, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, true, false, nil)
+ defer srv.Close()
hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
kp.User = perm
}
- kc, logbuf := runProxy(c, false, false, &kp)
- defer closeListener()
+ srv, kc, logbuf := runProxy(c, false, false, &kp)
+ defer srv.Close()
if admin {
kc.Arvados.ApiToken = arvadostest.AdminToken
} else {
c.Check(err, Equals, nil)
c.Log("Finished PutB (expected success)")
if admin {
- c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
+ c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload".* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* user_full_name="TestCase Administrator".* user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
} else {
- c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
+ c.Check(logbuf.String(), Matches, `(?ms).*msg="Block upload".* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* user_full_name="Active User".* user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
}
} else {
c.Check(hash2, Equals, "")
c.Check(blocklen, Equals, int64(3))
c.Log("Finished Get (expected success)")
if admin {
- c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="TestCase Administrator" user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
+ c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download".* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* user_full_name="TestCase Administrator".* user_uuid=zzzzz-tpzed-d9tiejq69daie8f.*`)
} else {
- c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download" locator=acbd18db4cc2f85cedef654fccc4a4d8\+3 user_full_name="Active User" user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
+ c.Check(logbuf.String(), Matches, `(?ms).*msg="Block download".* locator=acbd18db4cc2f85cedef654fccc4a4d8\+3.* user_full_name="Active User".* user_uuid=zzzzz-tpzed-xurymjxw79nv3jz.*`)
}
} else {
c.Check(err, FitsTypeOf, &keepclient.ErrNotFound{})
}
func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
- runProxy(c, false, false, nil)
- defer closeListener()
+ srv, _, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
{
client := http.Client{}
req, err := http.NewRequest("OPTIONS",
- fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))),
+ fmt.Sprintf("http://%s/%x+3", srv.Addr, md5.Sum([]byte("foo"))),
nil)
c.Assert(err, IsNil)
req.Header.Add("Access-Control-Request-Method", "PUT")
}
{
- resp, err := http.Get(
- fmt.Sprintf("http://%s/%x+3", listener.Addr().String(), md5.Sum([]byte("foo"))))
+ resp, err := http.Get(fmt.Sprintf("http://%s/%x+3", srv.Addr, md5.Sum([]byte("foo"))))
c.Check(err, Equals, nil)
c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
}
func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
- runProxy(c, false, false, nil)
- defer closeListener()
+ srv, _, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
{
client := http.Client{}
req, err := http.NewRequest("POST",
- "http://"+listener.Addr().String()+"/",
+ "http://"+srv.Addr+"/",
strings.NewReader("qux"))
c.Check(err, IsNil)
req.Header.Add("Authorization", "OAuth2 "+arvadostest.ActiveToken)
}
func getIndexWorker(c *C, useConfig bool) {
- kc, _ := runProxy(c, false, useConfig, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, useConfig, nil)
+ defer srv.Close()
// Put "index-data" blocks
data := []byte("index-data")
}
func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
hash, _, err := kc.PutB([]byte("shareddata"))
c.Check(err, IsNil)
kc.Arvados.ApiToken = arvadostest.FooCollectionSharingToken
}
func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
// Put a test block
hash, rep, err := kc.PutB([]byte("foo"))
}
func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
// Point keepproxy at a non-existent keepstore
locals := map[string]string{
TestProxyUUID: "http://localhost:12345",
}
- router.(*proxyHandler).KeepClient.SetServiceRoots(locals, nil, nil)
+ srv.proxyHandler.KeepClient.SetServiceRoots(locals, nil, nil)
// Ask should result in temporary bad gateway error
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
}
func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
for _, f := range []func() error{
}
func (s *ServerRequiredSuite) TestPing(c *C) {
- kc, _ := runProxy(c, false, false, nil)
- defer closeListener()
+ srv, kc, _ := runProxy(c, false, false, nil)
+ defer srv.Close()
- rtr, err := MakeRESTRouter(kc, 10*time.Second, &arvados.Cluster{ManagementToken: arvadostest.ManagementToken}, log.New())
+ rtr, err := newHandler(context.Background(), kc, 10*time.Second, &arvados.Cluster{ManagementToken: arvadostest.ManagementToken})
c.Assert(err, check.IsNil)
req, err := http.NewRequest("GET",
- "http://"+listener.Addr().String()+"/_health/ping",
+ "http://"+srv.Addr+"/_health/ping",
nil)
c.Assert(err, IsNil)
req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
//
// SPDX-License-Identifier: AGPL-3.0
-package main
+package keepproxy
import (
"net/http"
# SPDX-License-Identifier: AGPL-3.0
exec 2>&1
-sleep 2
set -ex -o pipefail
. /usr/local/lib/arvbox/common.sh
. /usr/local/lib/arvbox/go-setup.sh
-flock /var/lib/gopath/gopath.lock go install "git.arvados.org/arvados.git/services/keepproxy"
-install $GOPATH/bin/keepproxy /usr/local/bin
+(cd /usr/local/bin && ln -sf arvados-server keepproxy)
if test "$1" = "--only-deps" ; then
exit
fi
-exec /usr/local/bin/keepproxy
+/usr/local/lib/arvbox/runsu.sh flock $ARVADOS_CONTAINER_PATH/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+
+exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/keepproxy
projects / arvados.git / commitdiff