lib/crunchrun/container_gateway.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package crunchrun
   6
   7 import (
   8         "crypto/rand"
   9         "crypto/rsa"
  10         "fmt"
  11         "io"
  12         "net"
  13         "net/http"
  14         "os"
  15         "os/exec"
  16         "sync"
  17         "syscall"
  18
  19         "git.arvados.org/arvados.git/sdk/go/httpserver"
  20         "github.com/creack/pty"
  21         "github.com/google/shlex"
  22         "golang.org/x/crypto/ssh"
  23 )
  24
  25 // startGatewayServer starts an http server that allows authenticated
  26 // clients to open an interactive "docker exec" session and (in
  27 // future) connect to tcp ports inside the docker container.
  28 func (runner *ContainerRunner) startGatewayServer() error {
  29         runner.gatewaySSHConfig = &ssh.ServerConfig{
  30                 NoClientAuth: true,
  31                 PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
  32                         if c.User() == "_" {
  33                                 return nil, nil
  34                         } else {
  35                                 return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
  36                         }
  37                 },
  38                 PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
  39                         if c.User() == "_" {
  40                                 return &ssh.Permissions{
  41                                         Extensions: map[string]string{
  42                                                 "pubkey-fp": ssh.FingerprintSHA256(pubKey),
  43                                         },
  44                                 }, nil
  45                         } else {
  46                                 return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
  47                         }
  48                 },
  49         }
  50         pvt, err := rsa.GenerateKey(rand.Reader, 4096)
  51         if err != nil {
  52                 return err
  53         }
  54         err = pvt.Validate()
  55         if err != nil {
  56                 return err
  57         }
  58         signer, err := ssh.NewSignerFromKey(pvt)
  59         if err != nil {
  60                 return err
  61         }
  62         runner.gatewaySSHConfig.AddHostKey(signer)
  63
  64         // GatewayAddress (provided by arvados-dispatch-cloud) is
  65         // HOST:PORT where HOST is our IP address or hostname as seen
  66         // from arvados-controller, and PORT is either the desired
  67         // port where we should run our gateway server, or "0" if
  68         // we should choose an available port.
  69         host, port, err := net.SplitHostPort(os.Getenv("GatewayAddress"))
  70         if err != nil {
  71                 return err
  72         }
  73         srv := &httpserver.Server{
  74                 Server: http.Server{
  75                         Handler: http.HandlerFunc(runner.handleSSH),
  76                 },
  77                 Addr: ":" + port,
  78         }
  79         err = srv.Start()
  80         if err != nil {
  81                 return err
  82         }
  83         // Get the port number we are listening on (the port might be
  84         // "0" or a port name, in which case this will be different).
  85         _, port, err = net.SplitHostPort(srv.Addr)
  86         if err != nil {
  87                 return err
  88         }
  89         // When changing state to Running, we will set
  90         // gateway_address to "HOST:PORT" where HOST is our
  91         // external hostname/IP as provided by arvados-dispatch-cloud,
  92         // and PORT is the port number we ended up listening on.
  93         runner.gatewayAddress = net.JoinHostPort(host, port)
  94         return nil
  95 }
  96
  97 // handleSSH connects to an SSH server that runs commands as root in
  98 // the container. The tunnel itself can only be created by an
  99 // authenticated caller, so the SSH server itself is wide open (any
 100 // password or key will be accepted).
 101 //
 102 // Requests must have path "/ssh" and the following headers:
 103 //
 104 // Connection: upgrade
 105 // Upgrade: ssh
 106 // X-Arvados-Target-Uuid: uuid of container
 107 // X-Arvados-Authorization: must match GatewayAuthSecret provided by
 108 // a-d-c (this prevents other containers and shell nodes from
 109 // connecting directly)
 110 //
 111 // Optional header:
 112 //
 113 // X-Arvados-Detach-Keys: argument to "docker attach --detach-keys",
 114 // e.g., "ctrl-p,ctrl-q"
 115 func (runner *ContainerRunner) handleSSH(w http.ResponseWriter, req *http.Request) {
 116         // In future we'll handle browser traffic too, but for now the
 117         // only traffic we expect is an SSH tunnel from
 118         // (*lib/controller/localdb.Conn)ContainerSSH()
 119         if req.Method != "GET" || req.Header.Get("Upgrade") != "ssh" {
 120                 http.Error(w, "path not found", http.StatusNotFound)
 121                 return
 122         }
 123         if want := req.Header.Get("X-Arvados-Target-Uuid"); want != runner.Container.UUID {
 124                 http.Error(w, fmt.Sprintf("misdirected request: meant for %q but received by crunch-run %q", want, runner.Container.UUID), http.StatusBadGateway)
 125                 return
 126         }
 127         if req.Header.Get("X-Arvados-Authorization") != runner.gatewayAuthSecret {
 128                 http.Error(w, "bad X-Arvados-Authorization header", http.StatusUnauthorized)
 129                 return
 130         }
 131         detachKeys := req.Header.Get("X-Arvados-Detach-Keys")
 132         username := req.Header.Get("X-Arvados-Login-Username")
 133         if username == "" {
 134                 username = "root"
 135         }
 136         hj, ok := w.(http.Hijacker)
 137         if !ok {
 138                 http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
 139                 return
 140         }
 141         netconn, _, err := hj.Hijack()
 142         if !ok {
 143                 http.Error(w, err.Error(), http.StatusInternalServerError)
 144                 return
 145         }
 146         defer netconn.Close()
 147         w.Header().Set("Connection", "upgrade")
 148         w.Header().Set("Upgrade", "ssh")
 149         netconn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\n"))
 150         w.Header().Write(netconn)
 151         netconn.Write([]byte("\r\n"))
 152
 153         ctx := req.Context()
 154
 155         conn, newchans, reqs, err := ssh.NewServerConn(netconn, runner.gatewaySSHConfig)
 156         if err != nil {
 157                 runner.CrunchLog.Printf("ssh.NewServerConn: %s", err)
 158                 return
 159         }
 160         defer conn.Close()
 161         go ssh.DiscardRequests(reqs)
 162         for newch := range newchans {
 163                 if newch.ChannelType() != "session" {
 164                         newch.Reject(ssh.UnknownChannelType, "unknown channel type")
 165                         continue
 166                 }
 167                 ch, reqs, err := newch.Accept()
 168                 if err != nil {
 169                         runner.CrunchLog.Printf("accept channel: %s", err)
 170                         return
 171                 }
 172                 var pty0, tty0 *os.File
 173                 go func() {
 174                         defer pty0.Close()
 175                         defer tty0.Close()
 176                         // Where to send errors/messages for the
 177                         // client to see
 178                         logw := io.Writer(ch.Stderr())
 179                         // How to end lines when sending
 180                         // errors/messages to the client (changes to
 181                         // \r\n when using a pty)
 182                         eol := "\n"
 183                         // Env vars to add to child process
 184                         termEnv := []string(nil)
 185                         for req := range reqs {
 186                                 ok := false
 187                                 switch req.Type {
 188                                 case "shell", "exec":
 189                                         ok = true
 190                                         var payload struct {
 191                                                 Command string
 192                                         }
 193                                         ssh.Unmarshal(req.Payload, &payload)
 194                                         execargs, err := shlex.Split(payload.Command)
 195                                         if err != nil {
 196                                                 fmt.Fprintf(logw, "error parsing supplied command: %s"+eol, err)
 197                                                 return
 198                                         }
 199                                         if len(execargs) == 0 {
 200                                                 execargs = []string{"/bin/bash", "-login"}
 201                                         }
 202                                         go func() {
 203                                                 cmd := exec.CommandContext(ctx, "docker", "exec", "-i", "--detach-keys="+detachKeys, "--user="+username)
 204                                                 cmd.Stdin = ch
 205                                                 cmd.Stdout = ch
 206                                                 cmd.Stderr = ch.Stderr()
 207                                                 if tty0 != nil {
 208                                                         cmd.Args = append(cmd.Args, "-t")
 209                                                         cmd.Stdin = tty0
 210                                                         cmd.Stdout = tty0
 211                                                         cmd.Stderr = tty0
 212                                                         var wg sync.WaitGroup
 213                                                         defer wg.Wait()
 214                                                         wg.Add(2)
 215                                                         go func() { io.Copy(ch, pty0); wg.Done() }()
 216                                                         go func() { io.Copy(pty0, ch); wg.Done() }()
 217                                                         // Send our own debug messages to tty as well.
 218                                                         logw = tty0
 219                                                 }
 220                                                 cmd.Args = append(cmd.Args, runner.ContainerID)
 221                                                 cmd.Args = append(cmd.Args, execargs...)
 222                                                 cmd.SysProcAttr = &syscall.SysProcAttr{
 223                                                         Setctty: tty0 != nil,
 224                                                         Setsid:  true,
 225                                                 }
 226                                                 cmd.Env = append(os.Environ(), termEnv...)
 227                                                 err := cmd.Run()
 228                                                 errClose := ch.CloseWrite()
 229                                                 var resp struct {
 230                                                         Status uint32
 231                                                 }
 232                                                 if err, ok := err.(*exec.ExitError); ok {
 233                                                         if status, ok := err.Sys().(syscall.WaitStatus); ok {
 234                                                                 resp.Status = uint32(status.ExitStatus())
 235                                                         }
 236                                                 }
 237                                                 if resp.Status == 0 && (err != nil || errClose != nil) {
 238                                                         resp.Status = 1
 239                                                 }
 240                                                 ch.SendRequest("exit-status", false, ssh.Marshal(&resp))
 241                                                 ch.Close()
 242                                         }()
 243                                 case "pty-req":
 244                                         eol = "\r\n"
 245                                         p, t, err := pty.Open()
 246                                         if err != nil {
 247                                                 fmt.Fprintf(ch.Stderr(), "pty failed: %s"+eol, err)
 248                                                 break
 249                                         }
 250                                         pty0, tty0 = p, t
 251                                         ok = true
 252                                         var payload struct {
 253                                                 Term string
 254                                                 Cols uint32
 255                                                 Rows uint32
 256                                                 X    uint32
 257                                                 Y    uint32
 258                                         }
 259                                         ssh.Unmarshal(req.Payload, &payload)
 260                                         termEnv = []string{"TERM=" + payload.Term, "USE_TTY=1"}
 261                                         err = pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
 262                                         if err != nil {
 263                                                 fmt.Fprintf(logw, "pty-req: setsize failed: %s"+eol, err)
 264                                         }
 265                                 case "window-change":
 266                                         var payload struct {
 267                                                 Cols uint32
 268                                                 Rows uint32
 269                                                 X    uint32
 270                                                 Y    uint32
 271                                         }
 272                                         ssh.Unmarshal(req.Payload, &payload)
 273                                         err := pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
 274                                         if err != nil {
 275                                                 fmt.Fprintf(logw, "window-change: setsize failed: %s"+eol, err)
 276                                                 break
 277                                         }
 278                                         ok = true
 279                                 case "env":
 280                                         // TODO: implement "env"
 281                                         // requests by setting env
 282                                         // vars in the docker-exec
 283                                         // command (not docker-exec's
 284                                         // own environment, which
 285                                         // would be a gaping security
 286                                         // hole).
 287                                 default:
 288                                         // fmt.Fprintf(logw, "declining %q req"+eol, req.Type)
 289                                 }
 290                                 if req.WantReply {
 291                                         req.Reply(ok, nil)
 292                                 }
 293                         }
 294                 }()
 295         }
 296 }