1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
27 "git.arvados.org/arvados.git/lib/controller/rpc"
28 "git.arvados.org/arvados.git/lib/service"
29 "git.arvados.org/arvados.git/lib/webdavfs"
30 "git.arvados.org/arvados.git/sdk/go/arvados"
31 "git.arvados.org/arvados.git/sdk/go/auth"
32 "git.arvados.org/arvados.git/sdk/go/ctxlog"
33 "git.arvados.org/arvados.git/sdk/go/httpserver"
34 keepweb "git.arvados.org/arvados.git/services/keep-web"
35 "github.com/hashicorp/yamux"
36 "golang.org/x/net/webdav"
40 forceProxyForTest = false
41 forceInternalURLForTest *arvados.URL
44 // ContainerRequestLog returns a WebDAV handler that reads logs from
45 // the indicated container request. It works by proxying the incoming
48 // - the container gateway, if there is an associated container that
51 // - a different controller process, if there is a running container
52 // whose gateway is accessible through a tunnel to a different
55 // - keep-web, if saved logs exist and there is no gateway (or the
56 // associated container is finished)
58 // - an empty-collection stub, if there is no gateway and no saved
61 // For an incoming request
63 // GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
65 // The upstream request may be to {c_uuid}'s container gateway
67 // GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
68 // X-Webdav-Prefix: /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}
69 // X-Webdav-Source: /log
71 // ...or the upstream request may be to keep-web (where {cr_log_uuid}
72 // is the container request log collection UUID)
74 // GET /arvados/v1/container_requests/{cr_uuid}/log/{c_uuid}{/c_log_path}
75 // Host: {cr_log_uuid}.internal
76 // X-Webdav-Prefix: /arvados/v1/container_requests/{cr_uuid}/log
77 // X-Arvados-Container-Uuid: {c_uuid}
79 // ...or the request may be handled locally using an empty-collection
81 func (conn *Conn) ContainerRequestLog(ctx context.Context, opts arvados.ContainerLogOptions) (http.Handler, error) {
82 if opts.Method == "OPTIONS" && opts.Header.Get("Access-Control-Request-Method") != "" {
83 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
84 if !keepweb.ServeCORSPreflight(w, opts.Header) {
85 // Inconceivable. We already checked
86 // for the only condition where
87 // ServeCORSPreflight returns false.
88 httpserver.Error(w, "unhandled CORS preflight request", http.StatusInternalServerError)
92 cr, err := conn.railsProxy.ContainerRequestGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "container_uuid", "log_uuid"}})
94 if se := httpserver.HTTPStatusError(nil); errors.As(err, &se) && se.HTTPStatus() == http.StatusUnauthorized {
95 // Hint to WebDAV client that we accept HTTP basic auth.
96 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
97 w.Header().Set("Www-Authenticate", "Basic realm=\"collections\"")
98 w.WriteHeader(http.StatusUnauthorized)
103 ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: cr.ContainerUUID, Select: []string{"uuid", "state", "gateway_address"}})
107 // .../log/{ctr.UUID} is a directory where the currently
108 // assigned container's log data [will] appear (as opposed to
109 // previous attempts in .../log/{previous_ctr_uuid}). Requests
110 // that are outside that directory, and requests on a
111 // non-running container, are proxied to keep-web instead of
112 // going through the container gateway system.
114 // Side note: a depth>1 directory tree listing starting at
115 // .../{cr_uuid}/log will only include subdirectories for
116 // finished containers, i.e., will not include a subdirectory
117 // with log data for a current (unfinished) container UUID.
118 // In order to access live logs, a client must look up the
119 // container_uuid field of the container request record, and
120 // explicitly request a path under .../{cr_uuid}/log/{c_uuid}.
121 if ctr.GatewayAddress == "" ||
122 (ctr.State != arvados.ContainerStateLocked && ctr.State != arvados.ContainerStateRunning) ||
123 !(opts.Path == "/"+ctr.UUID || strings.HasPrefix(opts.Path, "/"+ctr.UUID+"/")) {
124 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
125 conn.serveContainerRequestLogViaKeepWeb(opts, cr, w, r)
128 dial, arpc, err := conn.findGateway(ctx, ctr, opts.NoForward)
133 opts.NoForward = true
134 return arpc.ContainerRequestLog(ctx, opts)
136 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
137 r = r.WithContext(ctx)
138 var proxyReq *http.Request
140 var expectRespondAuth string
141 proxy := &httputil.ReverseProxy{
142 // Our custom Transport:
144 // - Uses a custom dialer to connect to the
145 // gateway (either directly or through a
146 // tunnel set up though ContainerTunnel)
148 // - Verifies the gateway's TLS certificate
149 // using X-Arvados-Authorization headers.
151 // This involves modifying the outgoing
152 // request header in DialTLSContext.
153 // (ReverseProxy certainly doesn't expect us
154 // to do this, but it works.)
155 Transport: &http.Transport{
156 DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
157 tlsconn, requestAuth, respondAuth, err := dial()
161 proxyReq.Header.Set("X-Arvados-Authorization", requestAuth)
162 expectRespondAuth = respondAuth
166 Director: func(r *http.Request) {
167 // Scheme/host of incoming r.URL are
168 // irrelevant now, and may even be
169 // missing. Host is ignored by our
170 // DialTLSContext, but we need a
171 // generic syntactically correct URL
172 // for net/http to work with.
173 r.URL.Scheme = "https"
174 r.URL.Host = "0.0.0.0:0"
175 r.Header.Set("X-Arvados-Container-Gateway-Uuid", ctr.UUID)
176 r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log/"+ctr.UUID)
177 r.Header.Set("X-Webdav-Source", "/log")
180 ModifyResponse: func(resp *http.Response) error {
181 if resp.Header.Get("X-Arvados-Authorization-Response") != expectRespondAuth {
182 // Note this is how we detect
183 // an attacker-in-the-middle.
184 return httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
186 resp.Header.Del("X-Arvados-Authorization-Response")
187 for hdr := range resp.Header {
188 // proxy.ServeHTTP adds each
189 // resp.Header to w.Header,
190 // which causes duplicate CORS
191 // and request-id headers,
192 // unless we do this.
197 ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
201 proxy.ServeHTTP(w, r)
206 // If proxying to the container gateway fails, it
207 // might be caused by a race where crunch-run exited
208 // after we decided (above) the log was not final.
209 // In that case we should proxy to keep-web.
210 ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{
212 Select: []string{"uuid", "state", "gateway_address", "log"},
215 // Lost access to the container record?
216 httpserver.Error(w, "error re-fetching container record: "+err.Error(), http.StatusServiceUnavailable)
217 } else if ctr.State == arvados.ContainerStateLocked || ctr.State == arvados.ContainerStateRunning {
218 // No race, proxyErr was the best we can do
219 httpserver.Error(w, "proxy error: "+proxyErr.Error(), http.StatusServiceUnavailable)
221 conn.serveContainerRequestLogViaKeepWeb(opts, cr, w, r)
226 // serveContainerLogViaKeepWeb handles a request for saved container
227 // log content by proxying to one of the configured keep-web servers.
229 // It tries to choose a keep-web server that is running on this host.
230 func (conn *Conn) serveContainerRequestLogViaKeepWeb(opts arvados.ContainerLogOptions, cr arvados.ContainerRequest, w http.ResponseWriter, r *http.Request) {
231 if cr.LogUUID == "" {
232 // Special case: if no log data exists yet, we serve
233 // an empty collection by ourselves instead of
234 // proxying to keep-web.
235 conn.serveEmptyDir("/arvados/v1/container_requests/"+cr.UUID+"/log", w, r)
238 myURL, _ := service.URLFromContext(r.Context())
240 myHostname := u.Hostname()
241 var webdavBase arvados.URL
243 for webdavBase = range conn.cluster.Services.WebDAV.InternalURLs {
245 u := url.URL(webdavBase)
246 if h := u.Hostname(); h == "127.0.0.1" || h == "0.0.0.0" || h == "::1" || h == myHostname {
247 // Prefer a keep-web service running on the
248 // same host as us. (If we don't find one, we
249 // pick one arbitrarily.)
254 httpserver.Error(w, "no internalURLs configured for WebDAV service", http.StatusInternalServerError)
257 proxy := &httputil.ReverseProxy{
258 Director: func(r *http.Request) {
259 r.URL.Scheme = webdavBase.Scheme
260 r.URL.Host = webdavBase.Host
261 // Outgoing Host header specifies the
263 r.Host = cr.LogUUID + ".internal"
264 // We already checked permission on the
265 // container, so we can use a root token here
266 // instead of counting on the "access to log
267 // via container request and container"
268 // permission check, which can be racy when a
269 // request gets retried with a new container.
270 r.Header.Set("Authorization", "Bearer "+conn.cluster.SystemRootToken)
271 // We can't change r.URL.Path without
272 // confusing WebDAV (request body and response
273 // headers refer to the same paths) so we tell
274 // keep-web to map the log collection onto the
275 // containers/X/log/ namespace.
276 r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log")
277 if len(opts.Path) >= 28 && opts.Path[6:13] == "-dz642-" {
278 // "/arvados/v1/container_requests/{crUUID}/log/{cUUID}..."
280 // "/log for container {cUUID}..."
281 r.Header.Set("X-Webdav-Prefix", "/arvados/v1/container_requests/"+cr.UUID+"/log/"+opts.Path[1:28])
282 r.Header.Set("X-Webdav-Source", "/log for container "+opts.Path[1:28]+"/")
285 ModifyResponse: func(resp *http.Response) error {
286 for hdr := range resp.Header {
287 // proxy.ServeHTTP adds each
288 // resp.Header to w.Header, which
289 // causes duplicate CORS and
290 // request-id headers, unless we do
297 if conn.cluster.TLS.Insecure {
298 proxy.Transport = &http.Transport{
299 TLSClientConfig: &tls.Config{
300 InsecureSkipVerify: conn.cluster.TLS.Insecure,
304 proxy.ServeHTTP(w, r)
307 // serveEmptyDir handles read-only webdav requests as if there was an
308 // empty collection rooted at the given path. It's equivalent to
309 // proxying to an empty collection in keep-web, but avoids the extra
311 func (conn *Conn) serveEmptyDir(path string, w http.ResponseWriter, r *http.Request) {
312 wh := webdav.Handler{
314 FileSystem: webdav.NewMemFS(),
315 LockSystem: webdavfs.NoLockSystem,
316 Logger: func(r *http.Request, err error) {
317 if err != nil && !os.IsNotExist(err) {
318 ctxlog.FromContext(r.Context()).WithError(err).Info("webdav error on empty collection fs")
325 // ContainerSSH returns a connection to the SSH server in the
326 // appropriate crunch-run process on the worker node where the
327 // specified container is running.
329 // If the returned error is nil, the caller is responsible for closing
331 func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOptions) (sshconn arvados.ConnectionResponse, err error) {
332 user, err := conn.railsProxy.UserGetCurrent(ctx, arvados.GetOptions{})
336 ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "state", "gateway_address", "interactive_session_started"}})
340 ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{conn.cluster.SystemRootToken}})
341 if !user.IsAdmin || !conn.cluster.Containers.ShellAccess.Admin {
342 if !conn.cluster.Containers.ShellAccess.User {
343 return sshconn, httpserver.ErrorWithStatus(errors.New("shell access is disabled in config"), http.StatusServiceUnavailable)
345 crs, err := conn.railsProxy.ContainerRequestList(ctxRoot, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"container_uuid", "=", opts.UUID}}})
349 for _, cr := range crs.Items {
350 if cr.ModifiedByUserUUID != user.UUID {
351 return sshconn, httpserver.ErrorWithStatus(errors.New("permission denied: container is associated with requests submitted by other users"), http.StatusForbidden)
354 if crs.ItemsAvailable != len(crs.Items) {
355 return sshconn, httpserver.ErrorWithStatus(errors.New("incomplete response while checking permission"), http.StatusInternalServerError)
359 if ctr.State == arvados.ContainerStateQueued || ctr.State == arvados.ContainerStateLocked {
360 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("container is not running yet (state is %q)", ctr.State), http.StatusServiceUnavailable)
361 } else if ctr.State != arvados.ContainerStateRunning {
362 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("container has ended (state is %q)", ctr.State), http.StatusGone)
365 dial, arpc, err := conn.findGateway(ctx, ctr, opts.NoForward)
370 opts.NoForward = true
371 return arpc.ContainerSSH(ctx, opts)
374 tlsconn, requestAuth, respondAuth, err := dial()
378 bufr := bufio.NewReader(tlsconn)
379 bufw := bufio.NewWriter(tlsconn)
383 Host: tlsconn.RemoteAddr().String(),
386 postform := url.Values{
387 // uuid is only needed for older crunch-run versions
388 // (current version uses X-Arvados-* header below)
390 "detach_keys": {opts.DetachKeys},
391 "login_username": {opts.LoginUsername},
392 "no_forward": {fmt.Sprintf("%v", opts.NoForward)},
394 postdata := postform.Encode()
395 bufw.WriteString("POST " + u.String() + " HTTP/1.1\r\n")
396 bufw.WriteString("Host: " + u.Host + "\r\n")
397 bufw.WriteString("Upgrade: ssh\r\n")
398 bufw.WriteString("X-Arvados-Container-Gateway-Uuid: " + opts.UUID + "\r\n")
399 bufw.WriteString("X-Arvados-Authorization: " + requestAuth + "\r\n")
400 bufw.WriteString("Content-Type: application/x-www-form-urlencoded\r\n")
401 fmt.Fprintf(bufw, "Content-Length: %d\r\n", len(postdata))
402 bufw.WriteString("\r\n")
403 bufw.WriteString(postdata)
405 resp, err := http.ReadResponse(bufr, &http.Request{Method: "POST"})
408 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("error reading http response from gateway: %w", err), http.StatusBadGateway)
410 defer resp.Body.Close()
411 if resp.StatusCode != http.StatusSwitchingProtocols {
412 body, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 1000))
414 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("unexpected status %s %q", resp.Status, body), http.StatusBadGateway)
416 if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
417 strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
419 return sshconn, httpserver.ErrorWithStatus(errors.New("bad upgrade"), http.StatusBadGateway)
421 if resp.Header.Get("X-Arvados-Authorization-Response") != respondAuth {
423 return sshconn, httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
426 if !ctr.InteractiveSessionStarted {
427 _, err = conn.railsProxy.ContainerUpdate(ctxRoot, arvados.UpdateOptions{
429 Attrs: map[string]interface{}{
430 "interactive_session_started": true,
435 return sshconn, httpserver.ErrorWithStatus(err, http.StatusInternalServerError)
439 sshconn.Conn = tlsconn
440 sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
441 sshconn.Logger = ctxlog.FromContext(ctx)
442 sshconn.Header = http.Header{"Upgrade": {"ssh"}}
446 // ContainerGatewayTunnel sets up a tunnel enabling us (controller) to
447 // connect to the caller's (crunch-run's) gateway server.
448 func (conn *Conn) ContainerGatewayTunnel(ctx context.Context, opts arvados.ContainerGatewayTunnelOptions) (resp arvados.ConnectionResponse, err error) {
449 h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
450 fmt.Fprint(h, opts.UUID)
451 authSecret := fmt.Sprintf("%x", h.Sum(nil))
452 if subtle.ConstantTimeCompare([]byte(authSecret), []byte(opts.AuthSecret)) != 1 {
453 ctxlog.FromContext(ctx).Info("received incorrect auth_secret")
454 return resp, httpserver.ErrorWithStatus(errors.New("authentication error"), http.StatusUnauthorized)
457 muxconn, clientconn := net.Pipe()
458 tunnel, err := yamux.Server(muxconn, nil)
461 return resp, httpserver.ErrorWithStatus(err, http.StatusInternalServerError)
464 conn.gwTunnelsLock.Lock()
465 if conn.gwTunnels == nil {
466 conn.gwTunnels = map[string]*yamux.Session{opts.UUID: tunnel}
468 conn.gwTunnels[opts.UUID] = tunnel
470 conn.gwTunnelsLock.Unlock()
474 conn.gwTunnelsLock.Lock()
475 if conn.gwTunnels[opts.UUID] == tunnel {
476 delete(conn.gwTunnels, opts.UUID)
478 conn.gwTunnelsLock.Unlock()
481 // Assuming we're acting as the backend of an http server,
482 // lib/controller/router will call resp's ServeHTTP handler,
483 // which upgrades the incoming http connection to a raw socket
484 // and connects it to our yamux.Server through our net.Pipe().
485 resp.Conn = clientconn
486 resp.Bufrw = &bufio.ReadWriter{Reader: bufio.NewReader(&bytes.Buffer{}), Writer: bufio.NewWriter(&bytes.Buffer{})}
487 resp.Logger = ctxlog.FromContext(ctx)
488 resp.Header = http.Header{"Upgrade": {"tunnel"}}
489 if u, ok := service.URLFromContext(ctx); ok {
490 resp.Header.Set("X-Arvados-Internal-Url", u.String())
491 } else if forceInternalURLForTest != nil {
492 resp.Header.Set("X-Arvados-Internal-Url", forceInternalURLForTest.String())
497 type gatewayDialer func() (conn net.Conn, requestAuth, respondAuth string, err error)
499 // findGateway figures out how to connect to ctr's gateway.
501 // If the gateway can be contacted directly or through a tunnel on
502 // this instance, the first return value is a non-nil dialer.
504 // If the gateway is only accessible through a tunnel through a
505 // different controller process, the second return value is a non-nil
506 // *rpc.Conn for that controller.
507 func (conn *Conn) findGateway(ctx context.Context, ctr arvados.Container, noForward bool) (gatewayDialer, *rpc.Conn, error) {
508 conn.gwTunnelsLock.Lock()
509 tunnel := conn.gwTunnels[ctr.UUID]
510 conn.gwTunnelsLock.Unlock()
512 myURL, _ := service.URLFromContext(ctx)
514 if host, _, splitErr := net.SplitHostPort(ctr.GatewayAddress); splitErr == nil && host != "" && host != "127.0.0.1" {
515 // If crunch-run provided a GatewayAddress like
516 // "ipaddr:port", that means "ipaddr" is one of the
517 // external interfaces where the gateway is
518 // listening. In that case, it's the most
519 // reliable/direct option, so we use it even if a
520 // tunnel might also be available.
521 return func() (net.Conn, string, string, error) {
522 rawconn, err := (&net.Dialer{}).DialContext(ctx, "tcp", ctr.GatewayAddress)
524 return nil, "", "", httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
526 return conn.dialGatewayTLS(ctx, ctr, rawconn)
529 if tunnel != nil && !(forceProxyForTest && !noForward) {
530 // If we can't connect directly, and the gateway has
531 // established a yamux tunnel with us, connect through
534 // ...except: forceProxyForTest means we are emulating
535 // a situation where the gateway has established a
536 // yamux tunnel with controller B, and the
537 // ContainerSSH request arrives at controller A. If
538 // noForward==false then we are acting as A, so
539 // we pretend not to have a tunnel, and fall through
540 // to the "tunurl" case below. If noForward==true
541 // then the client is A and we are acting as B, so we
542 // connect to our tunnel.
543 return func() (net.Conn, string, string, error) {
544 rawconn, err := tunnel.Open()
546 return nil, "", "", httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
548 return conn.dialGatewayTLS(ctx, ctr, rawconn)
551 if tunurl := strings.TrimPrefix(ctr.GatewayAddress, "tunnel "); tunurl != ctr.GatewayAddress &&
553 tunurl != myURL.String() &&
555 // If crunch-run provided a GatewayAddress like
556 // "tunnel https://10.0.0.10:1010/", that means the
557 // gateway has established a yamux tunnel with the
558 // controller process at the indicated InternalURL
559 // (which isn't us, otherwise we would have had
560 // "tunnel != nil" above). We need to proxy through to
561 // the other controller process in order to use the
563 for u := range conn.cluster.Services.Controller.InternalURLs {
564 if u.String() == tunurl {
565 ctxlog.FromContext(ctx).Debugf("connecting to container gateway through other controller at %s", u)
567 return nil, rpc.NewConn(conn.cluster.ClusterID, &u, conn.cluster.TLS.Insecure, rpc.PassthroughTokenProvider), nil
570 ctxlog.FromContext(ctx).Warnf("container gateway provided a tunnel endpoint %s that is not one of Services.Controller.InternalURLs", tunurl)
571 return nil, nil, httpserver.ErrorWithStatus(errors.New("container gateway is running but tunnel endpoint is invalid"), http.StatusServiceUnavailable)
573 if ctr.GatewayAddress == "" {
574 return nil, nil, httpserver.ErrorWithStatus(errors.New("container is running but gateway is not available"), http.StatusServiceUnavailable)
576 return nil, nil, httpserver.ErrorWithStatus(errors.New("container is running but tunnel is down"), http.StatusServiceUnavailable)
580 // dialGatewayTLS negotiates a TLS connection to a container gateway
581 // over the given raw connection.
582 func (conn *Conn) dialGatewayTLS(ctx context.Context, ctr arvados.Container, rawconn net.Conn) (*tls.Conn, string, string, error) {
583 // crunch-run uses a self-signed / unverifiable TLS
584 // certificate, so we use the following scheme to ensure we're
585 // not talking to an attacker-in-the-middle.
587 // 1. Compute ctrKey = HMAC-SHA256(sysRootToken,ctrUUID) --
588 // this will be the same ctrKey that a-d-c supplied to
589 // crunch-run in the GatewayAuthSecret env var.
591 // 2. Compute requestAuth = HMAC-SHA256(ctrKey,serverCert) and
592 // send it to crunch-run as the X-Arvados-Authorization
593 // header, proving that we know ctrKey. (Note a MITM cannot
594 // replay the proof to a real crunch-run server, because the
595 // real crunch-run server would have a different cert.)
597 // 3. Compute respondAuth = HMAC-SHA256(ctrKey,requestAuth)
598 // and ensure the server returns it in the
599 // X-Arvados-Authorization-Response header, proving that the
600 // server knows ctrKey.
601 var requestAuth, respondAuth string
602 tlsconn := tls.Client(rawconn, &tls.Config{
603 InsecureSkipVerify: true,
604 VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
605 if len(rawCerts) == 0 {
606 return errors.New("no certificate received, cannot compute authorization header")
608 h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
609 fmt.Fprint(h, ctr.UUID)
610 authKey := fmt.Sprintf("%x", h.Sum(nil))
611 h = hmac.New(sha256.New, []byte(authKey))
613 requestAuth = fmt.Sprintf("%x", h.Sum(nil))
615 h.Write([]byte(requestAuth))
616 respondAuth = fmt.Sprintf("%x", h.Sum(nil))
620 err := tlsconn.HandshakeContext(ctx)
622 return nil, "", "", httpserver.ErrorWithStatus(fmt.Errorf("TLS handshake failed: %w", err), http.StatusBadGateway)
624 if respondAuth == "" {
626 return nil, "", "", httpserver.ErrorWithStatus(errors.New("BUG: no respondAuth"), http.StatusInternalServerError)
628 return tlsconn, requestAuth, respondAuth, nil