Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
config.lograge.formatter = Lograge::Formatters::Logstash.new
config.lograge.custom_options = lambda do |event|
payload = {
config.lograge.formatter = Lograge::Formatters::Logstash.new
config.lograge.custom_options = lambda do |event|
payload = {
+ ClusterID: Rails.configuration.ClusterID,
request_id: event.payload[:request_id],
}
# Also log params (minus the pseudo-params added by Rails). But if
request_id: event.payload[:request_id],
}
# Also log params (minus the pseudo-params added by Rails). But if
// logger with a new one according to the logging config.
log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel)
logger := log.WithFields(logrus.Fields{
// logger with a new one according to the logging config.
log = ctxlog.New(stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel)
logger := log.WithFields(logrus.Fields{
+ "PID": os.Getpid(),
+ "ClusterID": cluster.ClusterID,
})
ctx := ctxlog.Context(c.ctx, logger)
})
ctx := ctxlog.Context(c.ctx, logger)
config.lograge.formatter = Lograge::Formatters::Logstash.new
config.lograge.custom_options = lambda do |event|
payload = {
config.lograge.formatter = Lograge::Formatters::Logstash.new
config.lograge.custom_options = lambda do |event|
payload = {
+ ClusterID: Rails.configuration.ClusterID,
request_id: event.payload[:request_id],
client_ipaddr: event.payload[:client_ipaddr],
client_auth: event.payload[:client_auth],
request_id: event.payload[:request_id],
client_ipaddr: event.payload[:client_ipaddr],
client_auth: event.payload[:client_auth],
- logger := logrus.StandardLogger()
+ baseLogger := logrus.StandardLogger()
if os.Getenv("DEBUG") != "" {
if os.Getenv("DEBUG") != "" {
- logger.SetLevel(logrus.DebugLevel)
+ baseLogger.SetLevel(logrus.DebugLevel)
- logger.Formatter = &logrus.JSONFormatter{
+ baseLogger.Formatter = &logrus.JSONFormatter{
TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
}
TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00",
}
- loader := config.NewLoader(nil, logger)
+ loader := config.NewLoader(nil, baseLogger)
cfg, err := loader.Load()
if err != nil {
fmt.Fprintf(os.Stderr, "error loading config: %s\n", err)
cfg, err := loader.Load()
if err != nil {
fmt.Fprintf(os.Stderr, "error loading config: %s\n", err)
+ logger := baseLogger.WithField("ClusterID", cluster.ClusterID)
logger.Printf("crunch-dispatch-local %s started", version)
runningCmds = make(map[string]*exec.Cmd)
logger.Printf("crunch-dispatch-local %s started", version)
runningCmds = make(map[string]*exec.Cmd)
return fmt.Errorf("config error: %s", err)
}
return fmt.Errorf("config error: %s", err)
}
+ disp.logger = disp.logger.WithField("ClusterID", disp.cluster.ClusterID)
+
disp.Client.APIHost = disp.cluster.Services.Controller.ExternalURL.Host
disp.Client.AuthToken = disp.cluster.SystemRootToken
disp.Client.Insecure = disp.cluster.TLS.Insecure
disp.Client.APIHost = disp.cluster.Services.Controller.ExternalURL.Host
disp.Client.AuthToken = disp.cluster.SystemRootToken
disp.Client.Insecure = disp.cluster.TLS.Insecure
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/coreos/go-systemd/daemon"
"github.com/ghodss/yaml"
"github.com/sirupsen/logrus"
"github.com/coreos/go-systemd/daemon"
"github.com/ghodss/yaml"
"github.com/sirupsen/logrus"
+ initLogger := log.StandardLogger()
+ logger := initLogger.WithField("PID", os.Getpid())
cfg, err := configure(logger, os.Args)
if err != nil {
cfg, err := configure(logger, os.Args)
if err != nil {
} else if cfg == nil {
return
}
} else if cfg == nil {
return
}
-
- log.Printf("keep-web %s started", version)
+ logger = logger.WithField("ClusterID", cfg.cluster.ClusterID)
+ logger.Printf("keep-web %s started", version)
+ ctx := ctxlog.Context(context.Background(), logger)
if ext := ".txt"; mime.TypeByExtension(ext) == "" {
log.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext)
if ext := ".txt"; mime.TypeByExtension(ext) == "" {
log.Warnf("cannot look up MIME type for %q -- this probably means /etc/mime.types is missing -- clients will see incorrect content types", ext)
os.Setenv("ARVADOS_API_HOST", cfg.cluster.Services.Controller.ExternalURL.Host)
srv := &server{Config: cfg}
os.Setenv("ARVADOS_API_HOST", cfg.cluster.Services.Controller.ExternalURL.Host)
srv := &server{Config: cfg}
- if err := srv.Start(logrus.StandardLogger()); err != nil {
- log.Fatal(err)
+ if err := srv.Start(ctx, initLogger); err != nil {
+ logger.Fatal(err)
}
if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
}
if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.Printf("Error notifying init daemon: %v", err)
+ logger.Printf("Error notifying init daemon: %v", err)
- log.Println("Listening at", srv.Addr)
+ logger.Println("Listening at", srv.Addr)
if err := srv.Wait(); err != nil {
if err := srv.Wait(); err != nil {
"net/http"
"git.arvados.org/arvados.git/sdk/go/arvados"
"net/http"
"git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/prometheus/client_golang/prometheus"
"github.com/sirupsen/logrus"
-func (srv *server) Start(logger *logrus.Logger) error {
+func (srv *server) Start(ctx context.Context, logger *logrus.Logger) error {
h := &handler{Config: srv.Config}
reg := prometheus.NewRegistry()
h.Config.Cache.registry = reg
h := &handler{Config: srv.Config}
reg := prometheus.NewRegistry()
h.Config.Cache.registry = reg
mh := httpserver.Instrument(reg, logger, httpserver.AddRequestIDs(httpserver.LogRequests(h)))
h.MetricsAPI = mh.ServeAPI(h.Config.cluster.ManagementToken, http.NotFoundHandler())
srv.Handler = mh
mh := httpserver.Instrument(reg, logger, httpserver.AddRequestIDs(httpserver.LogRequests(h)))
h.MetricsAPI = mh.ServeAPI(h.Config.cluster.ManagementToken, http.NotFoundHandler())
srv.Handler = mh
- srv.BaseContext = func(net.Listener) context.Context { return ctxlog.Context(context.Background(), logger) }
+ srv.BaseContext = func(net.Listener) context.Context { return ctx }
var listen arvados.URL
for listen = range srv.Config.cluster.Services.WebDAV.InternalURLs {
break
var listen arvados.URL
for listen = range srv.Config.cluster.Services.WebDAV.InternalURLs {
break
"crypto/md5"
"encoding/json"
"fmt"
"crypto/md5"
"encoding/json"
"fmt"
cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.ArvConfig = arvCfg
s.testServer = &server{Config: cfg}
cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.ArvConfig = arvCfg
s.testServer = &server{Config: cfg}
- err = s.testServer.Start(ctxlog.TestLogger(c))
+ logger := ctxlog.TestLogger(c)
+ ctx := ctxlog.Context(context.Background(), logger)
+ err = s.testServer.Start(ctx, logger)
c.Assert(err, check.Equals, nil)
}
c.Assert(err, check.Equals, nil)
}
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"github.com/ghodss/yaml"
"github.com/gorilla/mux"
lru "github.com/hashicorp/golang-lru"
"github.com/ghodss/yaml"
"github.com/gorilla/mux"
lru "github.com/hashicorp/golang-lru"
- log "github.com/sirupsen/logrus"
+ "github.com/sirupsen/logrus"
const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-func configure(logger log.FieldLogger, args []string) (*arvados.Cluster, error) {
+func configure(args []string) (*arvados.Cluster, logrus.FieldLogger, error) {
prog := args[0]
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit")
getVersion := flags.Bool("version", false, "Print version information and exit.")
prog := args[0]
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
dumpConfig := flags.Bool("dump-config", false, "write current configuration to stdout and exit")
getVersion := flags.Bool("version", false, "Print version information and exit.")
+ initLogger := logrus.New()
+ initLogger.Formatter = &logrus.JSONFormatter{
+ TimestampFormat: rfc3339NanoFixed,
+ }
+ var logger logrus.FieldLogger = initLogger
+
loader := config.NewLoader(os.Stdin, logger)
loader.SetupFlags(flags)
args = loader.MungeLegacyConfigArgs(logger, args[1:], "-legacy-keepproxy-config")
loader := config.NewLoader(os.Stdin, logger)
loader.SetupFlags(flags)
args = loader.MungeLegacyConfigArgs(logger, args[1:], "-legacy-keepproxy-config")
os.Exit(code)
} else if *getVersion {
fmt.Printf("keepproxy %s\n", version)
os.Exit(code)
} else if *getVersion {
fmt.Printf("keepproxy %s\n", version)
+ return nil, logger, nil
}
cfg, err := loader.Load()
if err != nil {
}
cfg, err := loader.Load()
if err != nil {
+ return nil, logger, err
}
cluster, err := cfg.GetCluster("")
if err != nil {
}
cluster, err := cfg.GetCluster("")
if err != nil {
+ return nil, logger, err
+ logger = ctxlog.New(os.Stderr, cluster.SystemLogs.Format, cluster.SystemLogs.LogLevel).WithFields(logrus.Fields{
+ "ClusterID": cluster.ClusterID,
+ "PID": os.Getpid(),
+ })
+
if *dumpConfig {
out, err := yaml.Marshal(cfg)
if err != nil {
if *dumpConfig {
out, err := yaml.Marshal(cfg)
if err != nil {
+ return nil, logger, err
}
if _, err := os.Stdout.Write(out); err != nil {
}
if _, err := os.Stdout.Write(out); err != nil {
+ return nil, logger, err
+ return nil, logger, nil
+
+ return cluster, logger, nil
- logger := log.New()
- logger.Formatter = &log.JSONFormatter{
- TimestampFormat: rfc3339NanoFixed,
- }
-
- cluster, err := configure(logger, os.Args)
+ cluster, logger, err := configure(os.Args)
}
if cluster == nil {
return
}
}
if cluster == nil {
return
}
- log.Printf("keepproxy %s started", version)
+ logger.Printf("keepproxy %s started", version)
if err := run(logger, cluster); err != nil {
if err := run(logger, cluster); err != nil {
- log.Println("shutting down")
+ logger.Println("shutting down")
-func run(logger log.FieldLogger, cluster *arvados.Cluster) error {
+func run(logger logrus.FieldLogger, cluster *arvados.Cluster) error {
client, err := arvados.NewClientFromConfig(cluster)
if err != nil {
return err
client, err := arvados.NewClientFromConfig(cluster)
if err != nil {
return err
}
if cluster.SystemLogs.LogLevel == "debug" {
}
if cluster.SystemLogs.LogLevel == "debug" {
- keepclient.DebugPrintf = log.Printf
+ keepclient.DebugPrintf = logger.Printf
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
}
if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
}
if _, err := daemon.SdNotify(false, "READY=1"); err != nil {
- log.Printf("Error notifying init daemon: %v", err)
+ logger.Printf("Error notifying init daemon: %v", err)
- log.Println("listening at", listener.Addr())
+ logger.Println("listening at", listener.Addr())
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
term := make(chan os.Signal, 1)
go func(sig <-chan os.Signal) {
s := <-sig
// Shut down the server gracefully (by closing the listener)
// if SIGTERM is received.
term := make(chan os.Signal, 1)
go func(sig <-chan os.Signal) {
s := <-sig
- log.Println("caught signal:", s)
+ logger.Println("caught signal:", s)
listener.Close()
}(term)
signal.Notify(term, syscall.SIGTERM)
listener.Close()
}(term)
signal.Notify(term, syscall.SIGTERM)
if err != nil {
return err
}
if err != nil {
return err
}
- return http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
+ server := http.Server{
+ Handler: httpserver.AddRequestIDs(httpserver.LogRequests(router)),
+ BaseContext: func(net.Listener) context.Context {
+ return ctxlog.Context(context.Background(), logger)
+ },
+ }
+ return server.Serve(listener)
}
type TokenCacheEntry struct {
}
type TokenCacheEntry struct {
- log.Printf("%s: CheckAuthorizationHeader error: %v", GetRemoteAddress(req), err)
+ ctxlog.FromContext(req.Context()).Printf("%s: CheckAuthorizationHeader error: %v", GetRemoteAddress(req), err)
*APITokenCache
timeout time.Duration
transport *http.Transport
*APITokenCache
timeout time.Duration
transport *http.Transport
+ logger logrus.FieldLogger
cluster *arvados.Cluster
}
// MakeRESTRouter returns an http.Handler that passes GET and PUT
// requests to the appropriate handlers.
cluster *arvados.Cluster
}
// MakeRESTRouter returns an http.Handler that passes GET and PUT
// requests to the appropriate handlers.
-func MakeRESTRouter(kc *keepclient.KeepClient, timeout time.Duration, cluster *arvados.Cluster, logger log.FieldLogger) (http.Handler, error) {
+func MakeRESTRouter(kc *keepclient.KeepClient, timeout time.Duration, cluster *arvados.Cluster, logger logrus.FieldLogger) (http.Handler, error) {
rest := mux.NewRouter()
transport := defaultTransport
rest := mux.NewRouter()
transport := defaultTransport
type InvalidPathHandler struct{}
func (InvalidPathHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
type InvalidPathHandler struct{}
func (InvalidPathHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- log.Printf("%s: %s %s unroutable", GetRemoteAddress(req), req.Method, req.URL.Path)
+ ctxlog.FromContext(req.Context()).Printf("%s: %s %s unroutable", GetRemoteAddress(req), req.Method, req.URL.Path)
http.Error(resp, "Bad request", http.StatusBadRequest)
}
func (h *proxyHandler) Options(resp http.ResponseWriter, req *http.Request) {
http.Error(resp, "Bad request", http.StatusBadRequest)
}
func (h *proxyHandler) Options(resp http.ResponseWriter, req *http.Request) {
- log.Printf("%s: %s %s", GetRemoteAddress(req), req.Method, req.URL.Path)
+ ctxlog.FromContext(req.Context()).Printf("%s: %s %s", GetRemoteAddress(req), req.Method, req.URL.Path)
var proxiedURI = "-"
defer func() {
var proxiedURI = "-"
defer func() {
- log.Println(GetRemoteAddress(req), req.Method, req.URL.Path, status, expectLength, responseLength, proxiedURI, err)
+ h.logger.Println(GetRemoteAddress(req), req.Method, req.URL.Path, status, expectLength, responseLength, proxiedURI, err)
if status != http.StatusOK {
http.Error(resp, err.Error(), status)
}
if status != http.StatusOK {
http.Error(resp, err.Error(), status)
}
}
if expectLength == -1 {
}
if expectLength == -1 {
- log.Println("Warning:", GetRemoteAddress(req), req.Method, proxiedURI, "Content-Length not provided")
+ h.logger.Println("Warning:", GetRemoteAddress(req), req.Method, proxiedURI, "Content-Length not provided")
}
switch respErr := err.(type) {
}
switch respErr := err.(type) {
var locatorOut string = "-"
defer func() {
var locatorOut string = "-"
defer func() {
- log.Println(GetRemoteAddress(req), req.Method, req.URL.Path, status, expectLength, kc.Want_replicas, wroteReplicas, locatorOut, err)
+ h.logger.Println(GetRemoteAddress(req), req.Method, req.URL.Path, status, expectLength, kc.Want_replicas, wroteReplicas, locatorOut, err)
if status != http.StatusOK {
http.Error(resp, err.Error(), status)
}
if status != http.StatusOK {
http.Error(resp, err.Error(), status)
}