lib/controller/localdb/container_gateway.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bufio"
   9         "bytes"
  10         "context"
  11         "crypto/hmac"
  12         "crypto/sha256"
  13         "crypto/subtle"
  14         "crypto/tls"
  15         "crypto/x509"
  16         "errors"
  17         "fmt"
  18         "io"
  19         "io/ioutil"
  20         "net"
  21         "net/http"
  22         "net/http/httputil"
  23         "net/url"
  24         "strings"
  25
  26         "git.arvados.org/arvados.git/lib/controller/rpc"
  27         "git.arvados.org/arvados.git/lib/service"
  28         "git.arvados.org/arvados.git/lib/webdavfs"
  29         "git.arvados.org/arvados.git/sdk/go/arvados"
  30         "git.arvados.org/arvados.git/sdk/go/auth"
  31         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  32         "git.arvados.org/arvados.git/sdk/go/httpserver"
  33         "github.com/hashicorp/yamux"
  34         "golang.org/x/net/webdav"
  35 )
  36
  37 var (
  38         forceProxyForTest       = false
  39         forceInternalURLForTest *arvados.URL
  40 )
  41
  42 // ContainerLog returns a WebDAV handler that reads logs from the
  43 // indicated container. It works by proxying the request to
  44 //
  45 //   - the container gateway, if the container is running
  46 //
  47 //   - a different controller process, if the container is running and
  48 //     the gateway is accessible through a tunnel to a different
  49 //     controller process
  50 //
  51 //   - keep-web, if saved logs exist and there is no gateway (or the
  52 //     container is finished)
  53 //
  54 //   - an empty-collection stub, if there is no gateway and no saved
  55 //     log
  56 func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOptions) (http.Handler, error) {
  57         ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "state", "gateway_address", "log"}})
  58         if err != nil {
  59                 if se := httpserver.HTTPStatusError(nil); errors.As(err, &se) && se.HTTPStatus() == http.StatusUnauthorized {
  60                         // Hint to WebDAV client that we accept HTTP basic auth.
  61                         return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  62                                 w.Header().Set("Www-Authenticate", "Basic realm=\"collections\"")
  63                                 w.WriteHeader(http.StatusUnauthorized)
  64                         }), nil
  65                 }
  66                 return nil, err
  67         }
  68         if ctr.GatewayAddress == "" ||
  69                 (ctr.State != arvados.ContainerStateLocked && ctr.State != arvados.ContainerStateRunning) {
  70                 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  71                         conn.serveContainerLogViaKeepWeb(opts, ctr, w, r)
  72                 }), nil
  73         }
  74         dial, arpc, err := conn.findGateway(ctx, ctr, opts.NoForward)
  75         if err != nil {
  76                 return nil, err
  77         }
  78         if arpc != nil {
  79                 opts.NoForward = true
  80                 return arpc.ContainerLog(ctx, opts)
  81         }
  82         return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  83                 r = r.WithContext(ctx)
  84                 var proxyReq *http.Request
  85                 var proxyErr error
  86                 var expectRespondAuth string
  87                 proxy := &httputil.ReverseProxy{
  88                         // Our custom Transport:
  89                         //
  90                         // - Uses a custom dialer to connect to the
  91                         // gateway (either directly or through a
  92                         // tunnel set up though ContainerTunnel)
  93                         //
  94                         // - Verifies the gateway's TLS certificate
  95                         // using X-Arvados-Authorization headers.
  96                         //
  97                         // This involves modifying the outgoing
  98                         // request header in DialTLSContext.
  99                         // (ReverseProxy certainly doesn't expect us
 100                         // to do this, but it works.)
 101                         Transport: &http.Transport{
 102                                 DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 103                                         tlsconn, requestAuth, respondAuth, err := dial()
 104                                         if err != nil {
 105                                                 return nil, err
 106                                         }
 107                                         proxyReq.Header.Set("X-Arvados-Authorization", requestAuth)
 108                                         expectRespondAuth = respondAuth
 109                                         return tlsconn, nil
 110                                 },
 111                         },
 112                         Director: func(r *http.Request) {
 113                                 // Scheme/host of incoming r.URL are
 114                                 // irrelevant now, and may even be
 115                                 // missing. Host is ignored by our
 116                                 // DialTLSContext, but we need a
 117                                 // generic syntactically correct URL
 118                                 // for net/http to work with.
 119                                 r.URL.Scheme = "https"
 120                                 r.URL.Host = "0.0.0.0:0"
 121                                 r.Header.Set("X-Arvados-Container-Gateway-Uuid", opts.UUID)
 122                                 proxyReq = r
 123                         },
 124                         ModifyResponse: func(resp *http.Response) error {
 125                                 if resp.Header.Get("X-Arvados-Authorization-Response") != expectRespondAuth {
 126                                         // Note this is how we detect
 127                                         // an attacker-in-the-middle.
 128                                         return httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
 129                                 }
 130                                 return nil
 131                         },
 132                         ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
 133                                 proxyErr = err
 134                         },
 135                 }
 136                 proxy.ServeHTTP(w, r)
 137                 if proxyErr == nil {
 138                         // proxy succeeded
 139                         return
 140                 }
 141                 // If proxying to the container gateway fails, it
 142                 // might be caused by a race where crunch-run exited
 143                 // after we decided (above) the log was not final.
 144                 // In that case we should proxy to keep-web.
 145                 ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{
 146                         UUID:   opts.UUID,
 147                         Select: []string{"uuid", "state", "gateway_address", "log"},
 148                 })
 149                 if err != nil {
 150                         // Lost access to the container record?
 151                         httpserver.Error(w, "error re-fetching container record: "+err.Error(), http.StatusServiceUnavailable)
 152                 } else if ctr.State == arvados.ContainerStateLocked || ctr.State == arvados.ContainerStateRunning {
 153                         // No race, proxyErr was the best we can do
 154                         httpserver.Error(w, "proxy error: "+proxyErr.Error(), http.StatusServiceUnavailable)
 155                 } else {
 156                         conn.serveContainerLogViaKeepWeb(opts, ctr, w, r)
 157                 }
 158         }), nil
 159 }
 160
 161 // serveContainerLogViaKeepWeb handles a request for saved container
 162 // log content by proxying to one of the configured keep-web servers.
 163 //
 164 // It tries to choose a keep-web server that is running on this host.
 165 func (conn *Conn) serveContainerLogViaKeepWeb(opts arvados.ContainerLogOptions, ctr arvados.Container, w http.ResponseWriter, r *http.Request) {
 166         if ctr.Log == "" {
 167                 // Special case: if no log data exists yet, we serve
 168                 // an empty collection by ourselves instead of
 169                 // proxying to keep-web.
 170                 conn.serveEmptyDir("/arvados/v1/containers/"+ctr.UUID+"/log", w, r)
 171                 return
 172         }
 173         myURL, _ := service.URLFromContext(r.Context())
 174         u := url.URL(myURL)
 175         myHostname := u.Hostname()
 176         var webdavBase arvados.URL
 177         var ok bool
 178         for webdavBase = range conn.cluster.Services.WebDAVDownload.InternalURLs {
 179                 ok = true
 180                 u := url.URL(webdavBase)
 181                 if h := u.Hostname(); h == "127.0.0.1" || h == "0.0.0.0" || h == "::1" || h == myHostname {
 182                         // Prefer a keep-web service running on the
 183                         // same host as us. (If we don't find one, we
 184                         // pick one arbitrarily.)
 185                         break
 186                 }
 187         }
 188         if !ok {
 189                 httpserver.Error(w, "no internalURLs configured for WebDAV service", http.StatusInternalServerError)
 190                 return
 191         }
 192         proxy := &httputil.ReverseProxy{
 193                 Director: func(r *http.Request) {
 194                         r.URL.Scheme = webdavBase.Scheme
 195                         r.URL.Host = webdavBase.Host
 196                         // Outgoing Host header specifies the
 197                         // collection ID.
 198                         r.Host = strings.Replace(ctr.Log, "+", "-", -1) + ".internal"
 199                         // We already checked permission on the
 200                         // container, so we can use a root token here
 201                         // instead of counting on the "access to log
 202                         // via container request and container"
 203                         // permission check, which can be racy when a
 204                         // request gets retried with a new container.
 205                         r.Header.Set("Authorization", "Bearer "+conn.cluster.SystemRootToken)
 206                         // We can't change r.URL.Path without
 207                         // confusing WebDAV (request body and response
 208                         // headers refer to the same paths) so we tell
 209                         // keep-web to map the log collection onto the
 210                         // containers/X/log/ namespace.
 211                         r.Header.Set("X-Webdav-Prefix", "/arvados/v1/containers/"+ctr.UUID+"/log")
 212                 },
 213         }
 214         if conn.cluster.TLS.Insecure {
 215                 proxy.Transport = &http.Transport{
 216                         TLSClientConfig: &tls.Config{
 217                                 InsecureSkipVerify: conn.cluster.TLS.Insecure,
 218                         },
 219                 }
 220         }
 221         proxy.ServeHTTP(w, r)
 222 }
 223
 224 // serveEmptyDir handles read-only webdav requests as if there was an
 225 // empty collection rooted at the given path. It's equivalent to
 226 // proxying to an empty collection in keep-web, but avoids the extra
 227 // hop.
 228 func (conn *Conn) serveEmptyDir(path string, w http.ResponseWriter, r *http.Request) {
 229         wh := webdav.Handler{
 230                 Prefix:     path,
 231                 FileSystem: webdav.NewMemFS(),
 232                 LockSystem: webdavfs.NoLockSystem,
 233                 Logger: func(r *http.Request, err error) {
 234                         if err != nil {
 235                                 ctxlog.FromContext(r.Context()).WithError(err).Info("webdav error on empty collection fs")
 236                         }
 237                 },
 238         }
 239         wh.ServeHTTP(w, r)
 240 }
 241
 242 // ContainerSSH returns a connection to the SSH server in the
 243 // appropriate crunch-run process on the worker node where the
 244 // specified container is running.
 245 //
 246 // If the returned error is nil, the caller is responsible for closing
 247 // sshconn.Conn.
 248 func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOptions) (sshconn arvados.ConnectionResponse, err error) {
 249         user, err := conn.railsProxy.UserGetCurrent(ctx, arvados.GetOptions{})
 250         if err != nil {
 251                 return sshconn, err
 252         }
 253         ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "state", "gateway_address", "interactive_session_started"}})
 254         if err != nil {
 255                 return sshconn, err
 256         }
 257         ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{conn.cluster.SystemRootToken}})
 258         if !user.IsAdmin || !conn.cluster.Containers.ShellAccess.Admin {
 259                 if !conn.cluster.Containers.ShellAccess.User {
 260                         return sshconn, httpserver.ErrorWithStatus(errors.New("shell access is disabled in config"), http.StatusServiceUnavailable)
 261                 }
 262                 crs, err := conn.railsProxy.ContainerRequestList(ctxRoot, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"container_uuid", "=", opts.UUID}}})
 263                 if err != nil {
 264                         return sshconn, err
 265                 }
 266                 for _, cr := range crs.Items {
 267                         if cr.ModifiedByUserUUID != user.UUID {
 268                                 return sshconn, httpserver.ErrorWithStatus(errors.New("permission denied: container is associated with requests submitted by other users"), http.StatusForbidden)
 269                         }
 270                 }
 271                 if crs.ItemsAvailable != len(crs.Items) {
 272                         return sshconn, httpserver.ErrorWithStatus(errors.New("incomplete response while checking permission"), http.StatusInternalServerError)
 273                 }
 274         }
 275
 276         if ctr.State == arvados.ContainerStateQueued || ctr.State == arvados.ContainerStateLocked {
 277                 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("container is not running yet (state is %q)", ctr.State), http.StatusServiceUnavailable)
 278         } else if ctr.State != arvados.ContainerStateRunning {
 279                 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("container has ended (state is %q)", ctr.State), http.StatusGone)
 280         }
 281
 282         dial, arpc, err := conn.findGateway(ctx, ctr, opts.NoForward)
 283         if err != nil {
 284                 return sshconn, err
 285         }
 286         if arpc != nil {
 287                 opts.NoForward = true
 288                 return arpc.ContainerSSH(ctx, opts)
 289         }
 290
 291         tlsconn, requestAuth, respondAuth, err := dial()
 292         if err != nil {
 293                 return sshconn, err
 294         }
 295         bufr := bufio.NewReader(tlsconn)
 296         bufw := bufio.NewWriter(tlsconn)
 297
 298         u := url.URL{
 299                 Scheme: "http",
 300                 Host:   tlsconn.RemoteAddr().String(),
 301                 Path:   "/ssh",
 302         }
 303         postform := url.Values{
 304                 // uuid is only needed for older crunch-run versions
 305                 // (current version uses X-Arvados-* header below)
 306                 "uuid":           {opts.UUID},
 307                 "detach_keys":    {opts.DetachKeys},
 308                 "login_username": {opts.LoginUsername},
 309                 "no_forward":     {fmt.Sprintf("%v", opts.NoForward)},
 310         }
 311         postdata := postform.Encode()
 312         bufw.WriteString("POST " + u.String() + " HTTP/1.1\r\n")
 313         bufw.WriteString("Host: " + u.Host + "\r\n")
 314         bufw.WriteString("Upgrade: ssh\r\n")
 315         bufw.WriteString("X-Arvados-Container-Gateway-Uuid: " + opts.UUID + "\r\n")
 316         bufw.WriteString("X-Arvados-Authorization: " + requestAuth + "\r\n")
 317         bufw.WriteString("Content-Type: application/x-www-form-urlencoded\r\n")
 318         fmt.Fprintf(bufw, "Content-Length: %d\r\n", len(postdata))
 319         bufw.WriteString("\r\n")
 320         bufw.WriteString(postdata)
 321         bufw.Flush()
 322         resp, err := http.ReadResponse(bufr, &http.Request{Method: "POST"})
 323         if err != nil {
 324                 tlsconn.Close()
 325                 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("error reading http response from gateway: %w", err), http.StatusBadGateway)
 326         }
 327         defer resp.Body.Close()
 328         if resp.StatusCode != http.StatusSwitchingProtocols {
 329                 body, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 1000))
 330                 tlsconn.Close()
 331                 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("unexpected status %s %q", resp.Status, body), http.StatusBadGateway)
 332         }
 333         if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
 334                 strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
 335                 tlsconn.Close()
 336                 return sshconn, httpserver.ErrorWithStatus(errors.New("bad upgrade"), http.StatusBadGateway)
 337         }
 338         if resp.Header.Get("X-Arvados-Authorization-Response") != respondAuth {
 339                 tlsconn.Close()
 340                 return sshconn, httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
 341         }
 342
 343         if !ctr.InteractiveSessionStarted {
 344                 _, err = conn.railsProxy.ContainerUpdate(ctxRoot, arvados.UpdateOptions{
 345                         UUID: opts.UUID,
 346                         Attrs: map[string]interface{}{
 347                                 "interactive_session_started": true,
 348                         },
 349                 })
 350                 if err != nil {
 351                         tlsconn.Close()
 352                         return sshconn, httpserver.ErrorWithStatus(err, http.StatusInternalServerError)
 353                 }
 354         }
 355
 356         sshconn.Conn = tlsconn
 357         sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
 358         sshconn.Logger = ctxlog.FromContext(ctx)
 359         sshconn.Header = http.Header{"Upgrade": {"ssh"}}
 360         return sshconn, nil
 361 }
 362
 363 // ContainerGatewayTunnel sets up a tunnel enabling us (controller) to
 364 // connect to the caller's (crunch-run's) gateway server.
 365 func (conn *Conn) ContainerGatewayTunnel(ctx context.Context, opts arvados.ContainerGatewayTunnelOptions) (resp arvados.ConnectionResponse, err error) {
 366         h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
 367         fmt.Fprint(h, opts.UUID)
 368         authSecret := fmt.Sprintf("%x", h.Sum(nil))
 369         if subtle.ConstantTimeCompare([]byte(authSecret), []byte(opts.AuthSecret)) != 1 {
 370                 ctxlog.FromContext(ctx).Info("received incorrect auth_secret")
 371                 return resp, httpserver.ErrorWithStatus(errors.New("authentication error"), http.StatusUnauthorized)
 372         }
 373
 374         muxconn, clientconn := net.Pipe()
 375         tunnel, err := yamux.Server(muxconn, nil)
 376         if err != nil {
 377                 clientconn.Close()
 378                 return resp, httpserver.ErrorWithStatus(err, http.StatusInternalServerError)
 379         }
 380
 381         conn.gwTunnelsLock.Lock()
 382         if conn.gwTunnels == nil {
 383                 conn.gwTunnels = map[string]*yamux.Session{opts.UUID: tunnel}
 384         } else {
 385                 conn.gwTunnels[opts.UUID] = tunnel
 386         }
 387         conn.gwTunnelsLock.Unlock()
 388
 389         go func() {
 390                 <-tunnel.CloseChan()
 391                 conn.gwTunnelsLock.Lock()
 392                 if conn.gwTunnels[opts.UUID] == tunnel {
 393                         delete(conn.gwTunnels, opts.UUID)
 394                 }
 395                 conn.gwTunnelsLock.Unlock()
 396         }()
 397
 398         // Assuming we're acting as the backend of an http server,
 399         // lib/controller/router will call resp's ServeHTTP handler,
 400         // which upgrades the incoming http connection to a raw socket
 401         // and connects it to our yamux.Server through our net.Pipe().
 402         resp.Conn = clientconn
 403         resp.Bufrw = &bufio.ReadWriter{Reader: bufio.NewReader(&bytes.Buffer{}), Writer: bufio.NewWriter(&bytes.Buffer{})}
 404         resp.Logger = ctxlog.FromContext(ctx)
 405         resp.Header = http.Header{"Upgrade": {"tunnel"}}
 406         if u, ok := service.URLFromContext(ctx); ok {
 407                 resp.Header.Set("X-Arvados-Internal-Url", u.String())
 408         } else if forceInternalURLForTest != nil {
 409                 resp.Header.Set("X-Arvados-Internal-Url", forceInternalURLForTest.String())
 410         }
 411         return
 412 }
 413
 414 type gatewayDialer func() (conn net.Conn, requestAuth, respondAuth string, err error)
 415
 416 // findGateway figures out how to connect to ctr's gateway.
 417 //
 418 // If the gateway can be contacted directly or through a tunnel on
 419 // this instance, the first return value is a non-nil dialer.
 420 //
 421 // If the gateway is only accessible through a tunnel through a
 422 // different controller process, the second return value is a non-nil
 423 // *rpc.Conn for that controller.
 424 func (conn *Conn) findGateway(ctx context.Context, ctr arvados.Container, noForward bool) (gatewayDialer, *rpc.Conn, error) {
 425         conn.gwTunnelsLock.Lock()
 426         tunnel := conn.gwTunnels[ctr.UUID]
 427         conn.gwTunnelsLock.Unlock()
 428
 429         myURL, _ := service.URLFromContext(ctx)
 430
 431         if host, _, splitErr := net.SplitHostPort(ctr.GatewayAddress); splitErr == nil && host != "" && host != "127.0.0.1" {
 432                 // If crunch-run provided a GatewayAddress like
 433                 // "ipaddr:port", that means "ipaddr" is one of the
 434                 // external interfaces where the gateway is
 435                 // listening. In that case, it's the most
 436                 // reliable/direct option, so we use it even if a
 437                 // tunnel might also be available.
 438                 return func() (net.Conn, string, string, error) {
 439                         rawconn, err := (&net.Dialer{}).DialContext(ctx, "tcp", ctr.GatewayAddress)
 440                         if err != nil {
 441                                 err = httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
 442                         }
 443                         return conn.dialGatewayTLS(ctx, ctr, rawconn)
 444                 }, nil, nil
 445         }
 446         if tunnel != nil && !(forceProxyForTest && !noForward) {
 447                 // If we can't connect directly, and the gateway has
 448                 // established a yamux tunnel with us, connect through
 449                 // the tunnel.
 450                 //
 451                 // ...except: forceProxyForTest means we are emulating
 452                 // a situation where the gateway has established a
 453                 // yamux tunnel with controller B, and the
 454                 // ContainerSSH request arrives at controller A. If
 455                 // noForward==false then we are acting as A, so
 456                 // we pretend not to have a tunnel, and fall through
 457                 // to the "tunurl" case below. If noForward==true
 458                 // then the client is A and we are acting as B, so we
 459                 // connect to our tunnel.
 460                 return func() (net.Conn, string, string, error) {
 461                         rawconn, err := tunnel.Open()
 462                         if err != nil {
 463                                 err = httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
 464                         }
 465                         return conn.dialGatewayTLS(ctx, ctr, rawconn)
 466                 }, nil, nil
 467         }
 468         if tunurl := strings.TrimPrefix(ctr.GatewayAddress, "tunnel "); tunurl != ctr.GatewayAddress &&
 469                 tunurl != "" &&
 470                 tunurl != myURL.String() &&
 471                 !noForward {
 472                 // If crunch-run provided a GatewayAddress like
 473                 // "tunnel https://10.0.0.10:1010/", that means the
 474                 // gateway has established a yamux tunnel with the
 475                 // controller process at the indicated InternalURL
 476                 // (which isn't us, otherwise we would have had
 477                 // "tunnel != nil" above). We need to proxy through to
 478                 // the other controller process in order to use the
 479                 // tunnel.
 480                 for u := range conn.cluster.Services.Controller.InternalURLs {
 481                         if u.String() == tunurl {
 482                                 ctxlog.FromContext(ctx).Debugf("connecting to container gateway through other controller at %s", u)
 483                                 u := url.URL(u)
 484                                 return nil, rpc.NewConn(conn.cluster.ClusterID, &u, conn.cluster.TLS.Insecure, rpc.PassthroughTokenProvider), nil
 485                         }
 486                 }
 487                 ctxlog.FromContext(ctx).Warnf("container gateway provided a tunnel endpoint %s that is not one of Services.Controller.InternalURLs", tunurl)
 488                 return nil, nil, httpserver.ErrorWithStatus(errors.New("container gateway is running but tunnel endpoint is invalid"), http.StatusServiceUnavailable)
 489         }
 490         if ctr.GatewayAddress == "" {
 491                 return nil, nil, httpserver.ErrorWithStatus(errors.New("container is running but gateway is not available"), http.StatusServiceUnavailable)
 492         } else {
 493                 return nil, nil, httpserver.ErrorWithStatus(errors.New("container is running but tunnel is down"), http.StatusServiceUnavailable)
 494         }
 495 }
 496
 497 // dialGatewayTLS negotiates a TLS connection to a container gateway
 498 // over the given raw connection.
 499 func (conn *Conn) dialGatewayTLS(ctx context.Context, ctr arvados.Container, rawconn net.Conn) (*tls.Conn, string, string, error) {
 500         // crunch-run uses a self-signed / unverifiable TLS
 501         // certificate, so we use the following scheme to ensure we're
 502         // not talking to an attacker-in-the-middle.
 503         //
 504         // 1. Compute ctrKey = HMAC-SHA256(sysRootToken,ctrUUID) --
 505         // this will be the same ctrKey that a-d-c supplied to
 506         // crunch-run in the GatewayAuthSecret env var.
 507         //
 508         // 2. Compute requestAuth = HMAC-SHA256(ctrKey,serverCert) and
 509         // send it to crunch-run as the X-Arvados-Authorization
 510         // header, proving that we know ctrKey. (Note a MITM cannot
 511         // replay the proof to a real crunch-run server, because the
 512         // real crunch-run server would have a different cert.)
 513         //
 514         // 3. Compute respondAuth = HMAC-SHA256(ctrKey,requestAuth)
 515         // and ensure the server returns it in the
 516         // X-Arvados-Authorization-Response header, proving that the
 517         // server knows ctrKey.
 518         var requestAuth, respondAuth string
 519         tlsconn := tls.Client(rawconn, &tls.Config{
 520                 InsecureSkipVerify: true,
 521                 VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
 522                         if len(rawCerts) == 0 {
 523                                 return errors.New("no certificate received, cannot compute authorization header")
 524                         }
 525                         h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
 526                         fmt.Fprint(h, ctr.UUID)
 527                         authKey := fmt.Sprintf("%x", h.Sum(nil))
 528                         h = hmac.New(sha256.New, []byte(authKey))
 529                         h.Write(rawCerts[0])
 530                         requestAuth = fmt.Sprintf("%x", h.Sum(nil))
 531                         h.Reset()
 532                         h.Write([]byte(requestAuth))
 533                         respondAuth = fmt.Sprintf("%x", h.Sum(nil))
 534                         return nil
 535                 },
 536         })
 537         err := tlsconn.HandshakeContext(ctx)
 538         if err != nil {
 539                 return nil, "", "", httpserver.ErrorWithStatus(fmt.Errorf("TLS handshake failed: %w", err), http.StatusBadGateway)
 540         }
 541         if respondAuth == "" {
 542                 tlsconn.Close()
 543                 return nil, "", "", httpserver.ErrorWithStatus(errors.New("BUG: no respondAuth"), http.StatusInternalServerError)
 544         }
 545         return tlsconn, requestAuth, respondAuth, nil
 546 }