lib/controller/localdb/container_gateway.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bufio"
   9         "bytes"
  10         "context"
  11         "crypto/hmac"
  12         "crypto/sha256"
  13         "crypto/subtle"
  14         "crypto/tls"
  15         "crypto/x509"
  16         "errors"
  17         "fmt"
  18         "io"
  19         "io/ioutil"
  20         "net"
  21         "net/http"
  22         "net/http/httputil"
  23         "net/url"
  24         "os"
  25         "strings"
  26
  27         "git.arvados.org/arvados.git/lib/controller/rpc"
  28         "git.arvados.org/arvados.git/lib/service"
  29         "git.arvados.org/arvados.git/lib/webdavfs"
  30         "git.arvados.org/arvados.git/sdk/go/arvados"
  31         "git.arvados.org/arvados.git/sdk/go/auth"
  32         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  33         "git.arvados.org/arvados.git/sdk/go/httpserver"
  34         "github.com/hashicorp/yamux"
  35         "golang.org/x/net/webdav"
  36 )
  37
  38 var (
  39         forceProxyForTest       = false
  40         forceInternalURLForTest *arvados.URL
  41 )
  42
  43 // ContainerLog returns a WebDAV handler that reads logs from the
  44 // indicated container. It works by proxying the request to
  45 //
  46 //   - the container gateway, if the container is running
  47 //
  48 //   - a different controller process, if the container is running and
  49 //     the gateway is accessible through a tunnel to a different
  50 //     controller process
  51 //
  52 //   - keep-web, if saved logs exist and there is no gateway (or the
  53 //     container is finished)
  54 //
  55 //   - an empty-collection stub, if there is no gateway and no saved
  56 //     log
  57 func (conn *Conn) ContainerLog(ctx context.Context, opts arvados.ContainerLogOptions) (http.Handler, error) {
  58         ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "state", "gateway_address", "log"}})
  59         if err != nil {
  60                 if se := httpserver.HTTPStatusError(nil); errors.As(err, &se) && se.HTTPStatus() == http.StatusUnauthorized {
  61                         // Hint to WebDAV client that we accept HTTP basic auth.
  62                         return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  63                                 w.Header().Set("Www-Authenticate", "Basic realm=\"collections\"")
  64                                 w.WriteHeader(http.StatusUnauthorized)
  65                         }), nil
  66                 }
  67                 return nil, err
  68         }
  69         if ctr.GatewayAddress == "" ||
  70                 (ctr.State != arvados.ContainerStateLocked && ctr.State != arvados.ContainerStateRunning) {
  71                 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  72                         conn.serveContainerLogViaKeepWeb(opts, ctr, w, r)
  73                 }), nil
  74         }
  75         dial, arpc, err := conn.findGateway(ctx, ctr, opts.NoForward)
  76         if err != nil {
  77                 return nil, err
  78         }
  79         if arpc != nil {
  80                 opts.NoForward = true
  81                 return arpc.ContainerLog(ctx, opts)
  82         }
  83         return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  84                 r = r.WithContext(ctx)
  85                 var proxyReq *http.Request
  86                 var proxyErr error
  87                 var expectRespondAuth string
  88                 proxy := &httputil.ReverseProxy{
  89                         // Our custom Transport:
  90                         //
  91                         // - Uses a custom dialer to connect to the
  92                         // gateway (either directly or through a
  93                         // tunnel set up though ContainerTunnel)
  94                         //
  95                         // - Verifies the gateway's TLS certificate
  96                         // using X-Arvados-Authorization headers.
  97                         //
  98                         // This involves modifying the outgoing
  99                         // request header in DialTLSContext.
 100                         // (ReverseProxy certainly doesn't expect us
 101                         // to do this, but it works.)
 102                         Transport: &http.Transport{
 103                                 DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 104                                         tlsconn, requestAuth, respondAuth, err := dial()
 105                                         if err != nil {
 106                                                 return nil, err
 107                                         }
 108                                         proxyReq.Header.Set("X-Arvados-Authorization", requestAuth)
 109                                         expectRespondAuth = respondAuth
 110                                         return tlsconn, nil
 111                                 },
 112                         },
 113                         Director: func(r *http.Request) {
 114                                 // Scheme/host of incoming r.URL are
 115                                 // irrelevant now, and may even be
 116                                 // missing. Host is ignored by our
 117                                 // DialTLSContext, but we need a
 118                                 // generic syntactically correct URL
 119                                 // for net/http to work with.
 120                                 r.URL.Scheme = "https"
 121                                 r.URL.Host = "0.0.0.0:0"
 122                                 r.Header.Set("X-Arvados-Container-Gateway-Uuid", opts.UUID)
 123                                 proxyReq = r
 124                         },
 125                         ModifyResponse: func(resp *http.Response) error {
 126                                 if resp.Header.Get("X-Arvados-Authorization-Response") != expectRespondAuth {
 127                                         // Note this is how we detect
 128                                         // an attacker-in-the-middle.
 129                                         return httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
 130                                 }
 131                                 return nil
 132                         },
 133                         ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
 134                                 proxyErr = err
 135                         },
 136                 }
 137                 proxy.ServeHTTP(w, r)
 138                 if proxyErr == nil {
 139                         // proxy succeeded
 140                         return
 141                 }
 142                 // If proxying to the container gateway fails, it
 143                 // might be caused by a race where crunch-run exited
 144                 // after we decided (above) the log was not final.
 145                 // In that case we should proxy to keep-web.
 146                 ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{
 147                         UUID:   opts.UUID,
 148                         Select: []string{"uuid", "state", "gateway_address", "log"},
 149                 })
 150                 if err != nil {
 151                         // Lost access to the container record?
 152                         httpserver.Error(w, "error re-fetching container record: "+err.Error(), http.StatusServiceUnavailable)
 153                 } else if ctr.State == arvados.ContainerStateLocked || ctr.State == arvados.ContainerStateRunning {
 154                         // No race, proxyErr was the best we can do
 155                         httpserver.Error(w, "proxy error: "+proxyErr.Error(), http.StatusServiceUnavailable)
 156                 } else {
 157                         conn.serveContainerLogViaKeepWeb(opts, ctr, w, r)
 158                 }
 159         }), nil
 160 }
 161
 162 // serveContainerLogViaKeepWeb handles a request for saved container
 163 // log content by proxying to one of the configured keep-web servers.
 164 //
 165 // It tries to choose a keep-web server that is running on this host.
 166 func (conn *Conn) serveContainerLogViaKeepWeb(opts arvados.ContainerLogOptions, ctr arvados.Container, w http.ResponseWriter, r *http.Request) {
 167         if ctr.Log == "" {
 168                 // Special case: if no log data exists yet, we serve
 169                 // an empty collection by ourselves instead of
 170                 // proxying to keep-web.
 171                 conn.serveEmptyDir("/arvados/v1/containers/"+ctr.UUID+"/log", w, r)
 172                 return
 173         }
 174         myURL, _ := service.URLFromContext(r.Context())
 175         u := url.URL(myURL)
 176         myHostname := u.Hostname()
 177         var webdavBase arvados.URL
 178         var ok bool
 179         for webdavBase = range conn.cluster.Services.WebDAVDownload.InternalURLs {
 180                 ok = true
 181                 u := url.URL(webdavBase)
 182                 if h := u.Hostname(); h == "127.0.0.1" || h == "0.0.0.0" || h == "::1" || h == myHostname {
 183                         // Prefer a keep-web service running on the
 184                         // same host as us. (If we don't find one, we
 185                         // pick one arbitrarily.)
 186                         break
 187                 }
 188         }
 189         if !ok {
 190                 httpserver.Error(w, "no internalURLs configured for WebDAV service", http.StatusInternalServerError)
 191                 return
 192         }
 193         proxy := &httputil.ReverseProxy{
 194                 Director: func(r *http.Request) {
 195                         r.URL.Scheme = webdavBase.Scheme
 196                         r.URL.Host = webdavBase.Host
 197                         // Outgoing Host header specifies the
 198                         // collection ID.
 199                         r.Host = strings.Replace(ctr.Log, "+", "-", -1) + ".internal"
 200                         // We already checked permission on the
 201                         // container, so we can use a root token here
 202                         // instead of counting on the "access to log
 203                         // via container request and container"
 204                         // permission check, which can be racy when a
 205                         // request gets retried with a new container.
 206                         r.Header.Set("Authorization", "Bearer "+conn.cluster.SystemRootToken)
 207                         // We can't change r.URL.Path without
 208                         // confusing WebDAV (request body and response
 209                         // headers refer to the same paths) so we tell
 210                         // keep-web to map the log collection onto the
 211                         // containers/X/log/ namespace.
 212                         r.Header.Set("X-Webdav-Prefix", "/arvados/v1/containers/"+ctr.UUID+"/log")
 213                 },
 214         }
 215         if conn.cluster.TLS.Insecure {
 216                 proxy.Transport = &http.Transport{
 217                         TLSClientConfig: &tls.Config{
 218                                 InsecureSkipVerify: conn.cluster.TLS.Insecure,
 219                         },
 220                 }
 221         }
 222         proxy.ServeHTTP(w, r)
 223 }
 224
 225 // serveEmptyDir handles read-only webdav requests as if there was an
 226 // empty collection rooted at the given path. It's equivalent to
 227 // proxying to an empty collection in keep-web, but avoids the extra
 228 // hop.
 229 func (conn *Conn) serveEmptyDir(path string, w http.ResponseWriter, r *http.Request) {
 230         wh := webdav.Handler{
 231                 Prefix:     path,
 232                 FileSystem: webdav.NewMemFS(),
 233                 LockSystem: webdavfs.NoLockSystem,
 234                 Logger: func(r *http.Request, err error) {
 235                         if err != nil && !os.IsNotExist(err) {
 236                                 ctxlog.FromContext(r.Context()).WithError(err).Info("webdav error on empty collection fs")
 237                         }
 238                 },
 239         }
 240         wh.ServeHTTP(w, r)
 241 }
 242
 243 // ContainerSSH returns a connection to the SSH server in the
 244 // appropriate crunch-run process on the worker node where the
 245 // specified container is running.
 246 //
 247 // If the returned error is nil, the caller is responsible for closing
 248 // sshconn.Conn.
 249 func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOptions) (sshconn arvados.ConnectionResponse, err error) {
 250         user, err := conn.railsProxy.UserGetCurrent(ctx, arvados.GetOptions{})
 251         if err != nil {
 252                 return sshconn, err
 253         }
 254         ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID, Select: []string{"uuid", "state", "gateway_address", "interactive_session_started"}})
 255         if err != nil {
 256                 return sshconn, err
 257         }
 258         ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{conn.cluster.SystemRootToken}})
 259         if !user.IsAdmin || !conn.cluster.Containers.ShellAccess.Admin {
 260                 if !conn.cluster.Containers.ShellAccess.User {
 261                         return sshconn, httpserver.ErrorWithStatus(errors.New("shell access is disabled in config"), http.StatusServiceUnavailable)
 262                 }
 263                 crs, err := conn.railsProxy.ContainerRequestList(ctxRoot, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"container_uuid", "=", opts.UUID}}})
 264                 if err != nil {
 265                         return sshconn, err
 266                 }
 267                 for _, cr := range crs.Items {
 268                         if cr.ModifiedByUserUUID != user.UUID {
 269                                 return sshconn, httpserver.ErrorWithStatus(errors.New("permission denied: container is associated with requests submitted by other users"), http.StatusForbidden)
 270                         }
 271                 }
 272                 if crs.ItemsAvailable != len(crs.Items) {
 273                         return sshconn, httpserver.ErrorWithStatus(errors.New("incomplete response while checking permission"), http.StatusInternalServerError)
 274                 }
 275         }
 276
 277         if ctr.State == arvados.ContainerStateQueued || ctr.State == arvados.ContainerStateLocked {
 278                 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("container is not running yet (state is %q)", ctr.State), http.StatusServiceUnavailable)
 279         } else if ctr.State != arvados.ContainerStateRunning {
 280                 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("container has ended (state is %q)", ctr.State), http.StatusGone)
 281         }
 282
 283         dial, arpc, err := conn.findGateway(ctx, ctr, opts.NoForward)
 284         if err != nil {
 285                 return sshconn, err
 286         }
 287         if arpc != nil {
 288                 opts.NoForward = true
 289                 return arpc.ContainerSSH(ctx, opts)
 290         }
 291
 292         tlsconn, requestAuth, respondAuth, err := dial()
 293         if err != nil {
 294                 return sshconn, err
 295         }
 296         bufr := bufio.NewReader(tlsconn)
 297         bufw := bufio.NewWriter(tlsconn)
 298
 299         u := url.URL{
 300                 Scheme: "http",
 301                 Host:   tlsconn.RemoteAddr().String(),
 302                 Path:   "/ssh",
 303         }
 304         postform := url.Values{
 305                 // uuid is only needed for older crunch-run versions
 306                 // (current version uses X-Arvados-* header below)
 307                 "uuid":           {opts.UUID},
 308                 "detach_keys":    {opts.DetachKeys},
 309                 "login_username": {opts.LoginUsername},
 310                 "no_forward":     {fmt.Sprintf("%v", opts.NoForward)},
 311         }
 312         postdata := postform.Encode()
 313         bufw.WriteString("POST " + u.String() + " HTTP/1.1\r\n")
 314         bufw.WriteString("Host: " + u.Host + "\r\n")
 315         bufw.WriteString("Upgrade: ssh\r\n")
 316         bufw.WriteString("X-Arvados-Container-Gateway-Uuid: " + opts.UUID + "\r\n")
 317         bufw.WriteString("X-Arvados-Authorization: " + requestAuth + "\r\n")
 318         bufw.WriteString("Content-Type: application/x-www-form-urlencoded\r\n")
 319         fmt.Fprintf(bufw, "Content-Length: %d\r\n", len(postdata))
 320         bufw.WriteString("\r\n")
 321         bufw.WriteString(postdata)
 322         bufw.Flush()
 323         resp, err := http.ReadResponse(bufr, &http.Request{Method: "POST"})
 324         if err != nil {
 325                 tlsconn.Close()
 326                 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("error reading http response from gateway: %w", err), http.StatusBadGateway)
 327         }
 328         defer resp.Body.Close()
 329         if resp.StatusCode != http.StatusSwitchingProtocols {
 330                 body, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 1000))
 331                 tlsconn.Close()
 332                 return sshconn, httpserver.ErrorWithStatus(fmt.Errorf("unexpected status %s %q", resp.Status, body), http.StatusBadGateway)
 333         }
 334         if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
 335                 strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
 336                 tlsconn.Close()
 337                 return sshconn, httpserver.ErrorWithStatus(errors.New("bad upgrade"), http.StatusBadGateway)
 338         }
 339         if resp.Header.Get("X-Arvados-Authorization-Response") != respondAuth {
 340                 tlsconn.Close()
 341                 return sshconn, httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
 342         }
 343
 344         if !ctr.InteractiveSessionStarted {
 345                 _, err = conn.railsProxy.ContainerUpdate(ctxRoot, arvados.UpdateOptions{
 346                         UUID: opts.UUID,
 347                         Attrs: map[string]interface{}{
 348                                 "interactive_session_started": true,
 349                         },
 350                 })
 351                 if err != nil {
 352                         tlsconn.Close()
 353                         return sshconn, httpserver.ErrorWithStatus(err, http.StatusInternalServerError)
 354                 }
 355         }
 356
 357         sshconn.Conn = tlsconn
 358         sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
 359         sshconn.Logger = ctxlog.FromContext(ctx)
 360         sshconn.Header = http.Header{"Upgrade": {"ssh"}}
 361         return sshconn, nil
 362 }
 363
 364 // ContainerGatewayTunnel sets up a tunnel enabling us (controller) to
 365 // connect to the caller's (crunch-run's) gateway server.
 366 func (conn *Conn) ContainerGatewayTunnel(ctx context.Context, opts arvados.ContainerGatewayTunnelOptions) (resp arvados.ConnectionResponse, err error) {
 367         h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
 368         fmt.Fprint(h, opts.UUID)
 369         authSecret := fmt.Sprintf("%x", h.Sum(nil))
 370         if subtle.ConstantTimeCompare([]byte(authSecret), []byte(opts.AuthSecret)) != 1 {
 371                 ctxlog.FromContext(ctx).Info("received incorrect auth_secret")
 372                 return resp, httpserver.ErrorWithStatus(errors.New("authentication error"), http.StatusUnauthorized)
 373         }
 374
 375         muxconn, clientconn := net.Pipe()
 376         tunnel, err := yamux.Server(muxconn, nil)
 377         if err != nil {
 378                 clientconn.Close()
 379                 return resp, httpserver.ErrorWithStatus(err, http.StatusInternalServerError)
 380         }
 381
 382         conn.gwTunnelsLock.Lock()
 383         if conn.gwTunnels == nil {
 384                 conn.gwTunnels = map[string]*yamux.Session{opts.UUID: tunnel}
 385         } else {
 386                 conn.gwTunnels[opts.UUID] = tunnel
 387         }
 388         conn.gwTunnelsLock.Unlock()
 389
 390         go func() {
 391                 <-tunnel.CloseChan()
 392                 conn.gwTunnelsLock.Lock()
 393                 if conn.gwTunnels[opts.UUID] == tunnel {
 394                         delete(conn.gwTunnels, opts.UUID)
 395                 }
 396                 conn.gwTunnelsLock.Unlock()
 397         }()
 398
 399         // Assuming we're acting as the backend of an http server,
 400         // lib/controller/router will call resp's ServeHTTP handler,
 401         // which upgrades the incoming http connection to a raw socket
 402         // and connects it to our yamux.Server through our net.Pipe().
 403         resp.Conn = clientconn
 404         resp.Bufrw = &bufio.ReadWriter{Reader: bufio.NewReader(&bytes.Buffer{}), Writer: bufio.NewWriter(&bytes.Buffer{})}
 405         resp.Logger = ctxlog.FromContext(ctx)
 406         resp.Header = http.Header{"Upgrade": {"tunnel"}}
 407         if u, ok := service.URLFromContext(ctx); ok {
 408                 resp.Header.Set("X-Arvados-Internal-Url", u.String())
 409         } else if forceInternalURLForTest != nil {
 410                 resp.Header.Set("X-Arvados-Internal-Url", forceInternalURLForTest.String())
 411         }
 412         return
 413 }
 414
 415 type gatewayDialer func() (conn net.Conn, requestAuth, respondAuth string, err error)
 416
 417 // findGateway figures out how to connect to ctr's gateway.
 418 //
 419 // If the gateway can be contacted directly or through a tunnel on
 420 // this instance, the first return value is a non-nil dialer.
 421 //
 422 // If the gateway is only accessible through a tunnel through a
 423 // different controller process, the second return value is a non-nil
 424 // *rpc.Conn for that controller.
 425 func (conn *Conn) findGateway(ctx context.Context, ctr arvados.Container, noForward bool) (gatewayDialer, *rpc.Conn, error) {
 426         conn.gwTunnelsLock.Lock()
 427         tunnel := conn.gwTunnels[ctr.UUID]
 428         conn.gwTunnelsLock.Unlock()
 429
 430         myURL, _ := service.URLFromContext(ctx)
 431
 432         if host, _, splitErr := net.SplitHostPort(ctr.GatewayAddress); splitErr == nil && host != "" && host != "127.0.0.1" {
 433                 // If crunch-run provided a GatewayAddress like
 434                 // "ipaddr:port", that means "ipaddr" is one of the
 435                 // external interfaces where the gateway is
 436                 // listening. In that case, it's the most
 437                 // reliable/direct option, so we use it even if a
 438                 // tunnel might also be available.
 439                 return func() (net.Conn, string, string, error) {
 440                         rawconn, err := (&net.Dialer{}).DialContext(ctx, "tcp", ctr.GatewayAddress)
 441                         if err != nil {
 442                                 return nil, "", "", httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
 443                         }
 444                         return conn.dialGatewayTLS(ctx, ctr, rawconn)
 445                 }, nil, nil
 446         }
 447         if tunnel != nil && !(forceProxyForTest && !noForward) {
 448                 // If we can't connect directly, and the gateway has
 449                 // established a yamux tunnel with us, connect through
 450                 // the tunnel.
 451                 //
 452                 // ...except: forceProxyForTest means we are emulating
 453                 // a situation where the gateway has established a
 454                 // yamux tunnel with controller B, and the
 455                 // ContainerSSH request arrives at controller A. If
 456                 // noForward==false then we are acting as A, so
 457                 // we pretend not to have a tunnel, and fall through
 458                 // to the "tunurl" case below. If noForward==true
 459                 // then the client is A and we are acting as B, so we
 460                 // connect to our tunnel.
 461                 return func() (net.Conn, string, string, error) {
 462                         rawconn, err := tunnel.Open()
 463                         if err != nil {
 464                                 return nil, "", "", httpserver.ErrorWithStatus(err, http.StatusServiceUnavailable)
 465                         }
 466                         return conn.dialGatewayTLS(ctx, ctr, rawconn)
 467                 }, nil, nil
 468         }
 469         if tunurl := strings.TrimPrefix(ctr.GatewayAddress, "tunnel "); tunurl != ctr.GatewayAddress &&
 470                 tunurl != "" &&
 471                 tunurl != myURL.String() &&
 472                 !noForward {
 473                 // If crunch-run provided a GatewayAddress like
 474                 // "tunnel https://10.0.0.10:1010/", that means the
 475                 // gateway has established a yamux tunnel with the
 476                 // controller process at the indicated InternalURL
 477                 // (which isn't us, otherwise we would have had
 478                 // "tunnel != nil" above). We need to proxy through to
 479                 // the other controller process in order to use the
 480                 // tunnel.
 481                 for u := range conn.cluster.Services.Controller.InternalURLs {
 482                         if u.String() == tunurl {
 483                                 ctxlog.FromContext(ctx).Debugf("connecting to container gateway through other controller at %s", u)
 484                                 u := url.URL(u)
 485                                 return nil, rpc.NewConn(conn.cluster.ClusterID, &u, conn.cluster.TLS.Insecure, rpc.PassthroughTokenProvider), nil
 486                         }
 487                 }
 488                 ctxlog.FromContext(ctx).Warnf("container gateway provided a tunnel endpoint %s that is not one of Services.Controller.InternalURLs", tunurl)
 489                 return nil, nil, httpserver.ErrorWithStatus(errors.New("container gateway is running but tunnel endpoint is invalid"), http.StatusServiceUnavailable)
 490         }
 491         if ctr.GatewayAddress == "" {
 492                 return nil, nil, httpserver.ErrorWithStatus(errors.New("container is running but gateway is not available"), http.StatusServiceUnavailable)
 493         } else {
 494                 return nil, nil, httpserver.ErrorWithStatus(errors.New("container is running but tunnel is down"), http.StatusServiceUnavailable)
 495         }
 496 }
 497
 498 // dialGatewayTLS negotiates a TLS connection to a container gateway
 499 // over the given raw connection.
 500 func (conn *Conn) dialGatewayTLS(ctx context.Context, ctr arvados.Container, rawconn net.Conn) (*tls.Conn, string, string, error) {
 501         // crunch-run uses a self-signed / unverifiable TLS
 502         // certificate, so we use the following scheme to ensure we're
 503         // not talking to an attacker-in-the-middle.
 504         //
 505         // 1. Compute ctrKey = HMAC-SHA256(sysRootToken,ctrUUID) --
 506         // this will be the same ctrKey that a-d-c supplied to
 507         // crunch-run in the GatewayAuthSecret env var.
 508         //
 509         // 2. Compute requestAuth = HMAC-SHA256(ctrKey,serverCert) and
 510         // send it to crunch-run as the X-Arvados-Authorization
 511         // header, proving that we know ctrKey. (Note a MITM cannot
 512         // replay the proof to a real crunch-run server, because the
 513         // real crunch-run server would have a different cert.)
 514         //
 515         // 3. Compute respondAuth = HMAC-SHA256(ctrKey,requestAuth)
 516         // and ensure the server returns it in the
 517         // X-Arvados-Authorization-Response header, proving that the
 518         // server knows ctrKey.
 519         var requestAuth, respondAuth string
 520         tlsconn := tls.Client(rawconn, &tls.Config{
 521                 InsecureSkipVerify: true,
 522                 VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
 523                         if len(rawCerts) == 0 {
 524                                 return errors.New("no certificate received, cannot compute authorization header")
 525                         }
 526                         h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
 527                         fmt.Fprint(h, ctr.UUID)
 528                         authKey := fmt.Sprintf("%x", h.Sum(nil))
 529                         h = hmac.New(sha256.New, []byte(authKey))
 530                         h.Write(rawCerts[0])
 531                         requestAuth = fmt.Sprintf("%x", h.Sum(nil))
 532                         h.Reset()
 533                         h.Write([]byte(requestAuth))
 534                         respondAuth = fmt.Sprintf("%x", h.Sum(nil))
 535                         return nil
 536                 },
 537         })
 538         err := tlsconn.HandshakeContext(ctx)
 539         if err != nil {
 540                 return nil, "", "", httpserver.ErrorWithStatus(fmt.Errorf("TLS handshake failed: %w", err), http.StatusBadGateway)
 541         }
 542         if respondAuth == "" {
 543                 tlsconn.Close()
 544                 return nil, "", "", httpserver.ErrorWithStatus(errors.New("BUG: no respondAuth"), http.StatusInternalServerError)
 545         }
 546         return tlsconn, requestAuth, respondAuth, nil
 547 }