"net/http"
"net/http/httputil"
"net/url"
+ "os"
"strings"
"git.arvados.org/arvados.git/lib/controller/rpc"
forceInternalURLForTest *arvados.URL
)
-// ContainerLog returns a WebDAV handler that reads logs from the
-// indicated container. It works by proxying the request to
+// ContainerRequestLog returns a WebDAV handler that reads logs from
+// the indicated container request. It works by proxying the incoming
+// HTTP request to
//
-// - the container gateway, if the container is running
+// - the container gateway, if there is an associated container that
+// is running
//
-// - a different controller process, if the container is running and
-// the gateway is accessible through a tunnel to a different
+// - a different controller process, if there is a running container
+// whose gateway is accessible through a tunnel to a different
// controller process
//
// - keep-web, if saved logs exist and there is no gateway (or the
-// container is finished)
+// associated container is finished)
//
// - an empty-collection stub, if there is no gateway and no saved
// log
-func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOptions) (http.Handler, error) {
- ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "state", "gateway_address", "log"}})
+//
+// For an incoming request
+//
+// GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
+//
+// The upstream request may be to {c_uuid}'s container gateway
+//
+// GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
+// X-Webdav-Prefix: /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}
+// X-Webdav-Source: /log
+//
+// ...or the upstream request may be to keep-web (where {cr_log_uuid}
+// is the container request log collection UUID)
+//
+// GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
+// Host: {cr_log_uuid}.internal
+// X-Webdav-Prefix: /arvados/v1/container_requests/{cr_uuid}/log
+// X-Arvados-Container-Uuid: {c_uuid}
+//
+// ...or the request may be handled locally using an empty-collection
+// stub.
+func (conn *Conn) ContainerRequestLog(ctx context.Context, opts arvados.ContainerLogOptions) (http.Handler, error) {
+ cr, err := conn.railsProxy.ContainerRequestGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "container_uuid", "log_uuid"}})
if err != nil {
+ if se := httpserver.HTTPStatusError(nil); errors.As(err, &se) && se.HTTPStatus() == http.StatusUnauthorized {
+ // Hint to WebDAV client that we accept HTTP basic auth.
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Www-Authenticate", "Basic realm=\"collections\"")
+ w.WriteHeader(http.StatusUnauthorized)
+ }), nil
+ }
return nil, err
}
+ ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: cr.ContainerUUID, Select: []string{"uuid", "state", "gateway_address"}})
+ if err != nil {
+ return nil, err
+ }
+ // .../log/{ctr.UUID} is a directory where the currently
+ // assigned container's log data [will] appear (as opposed to
+ // previous attempts in .../log/{previous_ctr_uuid}). Requests
+ // that are outside that directory, and requests on a
+ // non-running container, are proxied to keep-web instead of
+ // going through the container gateway system.
+ //
+ // Side note: a depth>1 directory tree listing starting at
+ // .../{cr_uuid}/log will only include subdirectories for
+ // finished containers, i.e., will not include a subdirectory
+ // with log data for a current (unfinished) container UUID.
+ // In order to access live logs, a client must look up the
+ // container_uuid field of the container request record, and
+ // explicitly request a path under .../{cr_uuid}/log/{c_uuid}.
if ctr.GatewayAddress == "" ||
- (ctr.State != arvados.ContainerStateLocked && ctr.State != arvados.ContainerStateRunning) {
+ (ctr.State != arvados.ContainerStateLocked && ctr.State != arvados.ContainerStateRunning) ||
+ !(opts.Path == "/"+ctr.UUID || strings.HasPrefix(opts.Path, "/"+ctr.UUID+"/")) {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- conn.serveContainerLogViaKeepWeb(opts, ctr, w, r)
+ conn.serveContainerRequestLogViaKeepWeb(opts, cr, w, r)
}), nil
}
dial, arpc, err := conn.findGateway(ctx, ctr, opts.NoForward)
}
if arpc != nil {
opts.NoForward = true
- return arpc.ContainerLog(ctx, opts)
+ return arpc.ContainerRequestLog(ctx, opts)
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r = r.WithContext(ctx)
var proxyErr error
var expectRespondAuth string
proxy := &httputil.ReverseProxy{
+ // Our custom Transport:
+ //
+ // - Uses a custom dialer to connect to the
+ // gateway (either directly or through a
+ // tunnel set up though ContainerTunnel)
+ //
+ // - Verifies the gateway's TLS certificate
+ // using X-Arvados-Authorization headers.
+ //
+ // This involves modifying the outgoing
+ // request header in DialTLSContext.
+ // (ReverseProxy certainly doesn't expect us
+ // to do this, but it works.)
Transport: &http.Transport{
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
tlsconn, requestAuth, respondAuth, err := dial()
if err != nil {
return nil, err
}
- // Modify our response header
- // on the fly, even though
- // ReverseProxy surely doesn't
- // expect us to do this.
proxyReq.Header.Set("X-Arvados-Authorization", requestAuth)
expectRespondAuth = respondAuth
return tlsconn, nil
Director: func(r *http.Request) {
// Scheme/host of incoming r.URL are
// irrelevant now, and may even be
- // missing. Ensure we have a generic
- // syntactically correct URL for
- // net/http to work with. (Host is
- // ignored by our DialTLSContext.)
+ // missing. Host is ignored by our
+ // DialTLSContext, but we need a
+ // generic syntactically correct URL
+ // for net/http to work with.
r.URL.Scheme = "https"
r.URL.Host = "0.0.0.0:0"
- r.Header.Set("X-Arvados-Container-Gateway-Uuid", opts.UUID)
+ r.Header.Set("X-Arvados-Container-Gateway-Uuid", ctr.UUID)
+ r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log/"+ctr.UUID)
+ r.Header.Set("X-Webdav-Source", "/log")
proxyReq = r
},
ModifyResponse: func(resp *http.Response) error {
if resp.Header.Get("X-Arvados-Authorization-Response") != expectRespondAuth {
+ // Note this is how we detect
+ // an attacker-in-the-middle.
return httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
}
return nil
// after we decided (above) the log was not final.
// In that case we should proxy to keep-web.
ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{
- UUID: opts.UUID,
+ UUID: ctr.UUID,
Select: []string{"uuid", "state", "gateway_address", "log"},
})
if err != nil {
// No race, proxyErr was the best we can do
httpserver.Error(w, "proxy error: "+proxyErr.Error(), http.StatusServiceUnavailable)
} else {
- conn.serveContainerLogViaKeepWeb(opts, ctr, w, r)
+ conn.serveContainerRequestLogViaKeepWeb(opts, cr, w, r)
}
}), nil
}
// log content by proxying to one of the configured keep-web servers.
//
// It tries to choose a keep-web server that is running on this host.
-func (conn *Conn) serveContainerLogViaKeepWeb(opts arvados.ContainerLogOptions, ctr arvados.Container, w http.ResponseWriter, r *http.Request) {
- if ctr.Log == "" {
+func (conn *Conn) serveContainerRequestLogViaKeepWeb(opts arvados.ContainerLogOptions, cr arvados.ContainerRequest, w http.ResponseWriter, r *http.Request) {
+ if cr.LogUUID == "" {
// Special case: if no log data exists yet, we serve
// an empty collection by ourselves instead of
// proxying to keep-web.
- conn.serveEmptyDir("/arvados/v1/containers/"+ctr.UUID+"/log", w, r)
+ conn.serveEmptyDir("/arvados/v1/container_requests/"+cr.UUID+"/log", w, r)
return
}
myURL, _ := service.URLFromContext(r.Context())
}
proxy := &httputil.ReverseProxy{
Director: func(r *http.Request) {
- r.Host = conn.cluster.Services.WebDAVDownload.ExternalURL.Host
- r.URL = &url.URL{
- Scheme: webdavBase.Scheme,
- Host: webdavBase.Host,
- Path: "/by_id/" + url.PathEscape(ctr.Log) + opts.Path,
- }
+ r.URL.Scheme = webdavBase.Scheme
+ r.URL.Host = webdavBase.Host
+ // Outgoing Host header specifies the
+ // collection ID.
+ r.Host = cr.LogUUID + ".internal"
+ // We already checked permission on the
+ // container, so we can use a root token here
+ // instead of counting on the "access to log
+ // via container request and container"
+ // permission check, which can be racy when a
+ // request gets retried with a new container.
r.Header.Set("Authorization", "Bearer "+conn.cluster.SystemRootToken)
+ // We can't change r.URL.Path without
+ // confusing WebDAV (request body and response
+ // headers refer to the same paths) so we tell
+ // keep-web to map the log collection onto the
+ // containers/X/log/ namespace.
+ r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log")
+ if len(opts.Path) >= 28 && opts.Path[6:13] == "-dz642-" {
+ // "/arvados/v1/container_requests/{crUUID}/log/{cUUID}..."
+ // proxies to
+ // "/log for container {cUUID}..."
+ r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log/"+opts.Path[1:28])
+ r.Header.Set("X-Webdav-Source", "/log for container "+opts.Path[1:28]+"/")
+ }
},
}
if conn.cluster.TLS.Insecure {
FileSystem: webdav.NewMemFS(),
LockSystem: webdavfs.NoLockSystem,
Logger: func(r *http.Request, err error) {
- if err != nil {
+ if err != nil && !os.IsNotExist(err) {
ctxlog.FromContext(r.Context()).WithError(err).Info("webdav error on empty collection fs")
}
},
return func() (net.Conn, string, string, error) {
rawconn, err := (&net.Dialer{}).DialContext(ctx, "tcp", ctr.GatewayAddress)
if err != nil {
- err = httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
+ return nil, "", "", httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
}
return conn.dialGatewayTLS(ctx, ctr, rawconn)
}, nil, nil
return func() (net.Conn, string, string, error) {
rawconn, err := tunnel.Open()
if err != nil {
- err = httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
+ return nil, "", "", httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
}
return conn.dialGatewayTLS(ctx, ctr, rawconn)
}, nil, nil
}
}
+// dialGatewayTLS negotiates a TLS connection to a container gateway
+// over the given raw connection.
func (conn *Conn) dialGatewayTLS(ctx context.Context, ctr arvados.Container, rawconn net.Conn) (*tls.Conn, string, string, error) {
// crunch-run uses a self-signed / unverifiable TLS
// certificate, so we use the following scheme to ensure we're
- // not talking to a MITM.
+ // not talking to an attacker-in-the-middle.
//
// 1. Compute ctrKey = HMAC-SHA256(sysRootToken,ctrUUID) --
// this will be the same ctrKey that a-d-c supplied to