1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
22 "git.arvados.org/arvados.git/lib/selfsigned"
23 "git.arvados.org/arvados.git/sdk/go/ctxlog"
24 "git.arvados.org/arvados.git/sdk/go/httpserver"
25 "github.com/creack/pty"
26 "github.com/google/shlex"
27 "golang.org/x/crypto/ssh"
28 "golang.org/x/net/context"
31 type GatewayTarget interface {
32 // Command that will execute cmd inside the container
33 InjectCommand(ctx context.Context, detachKeys, username string, usingTTY bool, cmd []string) (*exec.Cmd, error)
35 // IP address inside container
36 IPAddress() (string, error)
41 Address string // listen host:port; if port=0, Start() will change it to the selected port
45 Printf(fmt string, args ...interface{})
48 sshConfig ssh.ServerConfig
53 // Start starts an http server that allows authenticated clients to open an
54 // interactive "docker exec" session and (in future) connect to tcp ports
55 // inside the docker container.
56 func (gw *Gateway) Start() error {
57 gw.sshConfig = ssh.ServerConfig{
59 PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
63 return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
65 PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
67 return &ssh.Permissions{
68 Extensions: map[string]string{
69 "pubkey-fp": ssh.FingerprintSHA256(pubKey),
73 return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
76 pvt, err := rsa.GenerateKey(rand.Reader, 2048)
84 signer, err := ssh.NewSignerFromKey(pvt)
88 gw.sshConfig.AddHostKey(signer)
90 // Address (typically provided by arvados-dispatch-cloud) is
91 // HOST:PORT where HOST is our IP address or hostname as seen
92 // from arvados-controller, and PORT is either the desired
93 // port where we should run our gateway server, or "0" if we
94 // should choose an available port.
95 host, port, err := net.SplitHostPort(gw.Address)
99 cert, err := selfsigned.CertGenerator{}.Generate()
103 h := hmac.New(sha256.New, []byte(gw.AuthSecret))
104 h.Write(cert.Certificate[0])
105 gw.requestAuth = fmt.Sprintf("%x", h.Sum(nil))
107 h.Write([]byte(gw.requestAuth))
108 gw.respondAuth = fmt.Sprintf("%x", h.Sum(nil))
110 srv := &httpserver.Server{
112 Handler: http.HandlerFunc(gw.handleSSH),
113 TLSConfig: &tls.Config{
114 Certificates: []tls.Certificate{cert},
123 // Get the port number we are listening on (the port might be
124 // "0" or a port name, in which case this will be different).
125 _, port, err = net.SplitHostPort(srv.Addr)
129 // When changing state to Running, we will set
130 // gateway_address to "HOST:PORT" where HOST is our
131 // external hostname/IP as provided by arvados-dispatch-cloud,
132 // and PORT is the port number we ended up listening on.
133 gw.Address = net.JoinHostPort(host, port)
137 // handleSSH connects to an SSH server that allows the caller to run
138 // interactive commands as root (or any other desired user) inside the
139 // container. The tunnel itself can only be created by an
140 // authenticated caller, so the SSH server itself is wide open (any
141 // password or key will be accepted).
143 // Requests must have path "/ssh" and the following headers:
145 // Connection: upgrade
147 // X-Arvados-Target-Uuid: uuid of container
148 // X-Arvados-Authorization: must match
149 // hmac(AuthSecret,certfingerprint) (this prevents other containers
150 // and shell nodes from connecting directly)
154 // X-Arvados-Detach-Keys: argument to "docker exec --detach-keys",
155 // e.g., "ctrl-p,ctrl-q"
156 // X-Arvados-Login-Username: argument to "docker exec --user": account
157 // used to run command(s) inside the container.
158 func (gw *Gateway) handleSSH(w http.ResponseWriter, req *http.Request) {
159 // In future we'll handle browser traffic too, but for now the
160 // only traffic we expect is an SSH tunnel from
161 // (*lib/controller/localdb.Conn)ContainerSSH()
162 if req.Method != "GET" || req.Header.Get("Upgrade") != "ssh" {
163 http.Error(w, "path not found", http.StatusNotFound)
166 if want := req.Header.Get("X-Arvados-Target-Uuid"); want != gw.ContainerUUID {
167 http.Error(w, fmt.Sprintf("misdirected request: meant for %q but received by crunch-run %q", want, gw.ContainerUUID), http.StatusBadGateway)
170 if req.Header.Get("X-Arvados-Authorization") != gw.requestAuth {
171 http.Error(w, "bad X-Arvados-Authorization header", http.StatusUnauthorized)
174 detachKeys := req.Header.Get("X-Arvados-Detach-Keys")
175 username := req.Header.Get("X-Arvados-Login-Username")
179 hj, ok := w.(http.Hijacker)
181 http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
184 netconn, _, err := hj.Hijack()
186 http.Error(w, err.Error(), http.StatusInternalServerError)
189 defer netconn.Close()
190 w.Header().Set("Connection", "upgrade")
191 w.Header().Set("Upgrade", "ssh")
192 w.Header().Set("X-Arvados-Authorization-Response", gw.respondAuth)
193 netconn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\n"))
194 w.Header().Write(netconn)
195 netconn.Write([]byte("\r\n"))
199 conn, newchans, reqs, err := ssh.NewServerConn(netconn, &gw.sshConfig)
201 gw.Log.Printf("ssh.NewServerConn: %s", err)
205 go ssh.DiscardRequests(reqs)
206 for newch := range newchans {
207 switch newch.ChannelType() {
209 go gw.handleDirectTCPIP(ctx, newch)
211 go gw.handleSession(ctx, newch, detachKeys, username)
213 go newch.Reject(ssh.UnknownChannelType, fmt.Sprintf("unsupported channel type %q", newch.ChannelType()))
218 func (gw *Gateway) handleDirectTCPIP(ctx context.Context, newch ssh.NewChannel) {
219 ch, reqs, err := newch.Accept()
221 gw.Log.Printf("accept direct-tcpip channel: %s", err)
225 go ssh.DiscardRequests(reqs)
227 // RFC 4254 7.2 (copy of channelOpenDirectMsg in
228 // golang.org/x/crypto/ssh)
235 err = ssh.Unmarshal(newch.ExtraData(), &msg)
237 fmt.Fprintf(ch.Stderr(), "unmarshal direct-tcpip extradata: %s\n", err)
241 case "localhost", "0.0.0.0", "127.0.0.1", "::1", "::":
243 fmt.Fprintf(ch.Stderr(), "cannot forward to ports on %q, only localhost\n", msg.Raddr)
247 dstaddr, err := gw.Target.IPAddress()
249 fmt.Fprintf(ch.Stderr(), "container has no IP address: %s\n", err)
251 } else if dstaddr == "" {
252 fmt.Fprintf(ch.Stderr(), "container has no IP address\n")
256 dst := net.JoinHostPort(dstaddr, fmt.Sprintf("%d", msg.Rport))
257 tcpconn, err := net.Dial("tcp", dst)
259 fmt.Fprintf(ch.Stderr(), "%s: %s\n", dst, err)
263 n, _ := io.Copy(ch, tcpconn)
264 ctxlog.FromContext(ctx).Debugf("tcpip: sent %d bytes\n", n)
267 n, _ := io.Copy(tcpconn, ch)
268 ctxlog.FromContext(ctx).Debugf("tcpip: received %d bytes\n", n)
271 func (gw *Gateway) handleSession(ctx context.Context, newch ssh.NewChannel, detachKeys, username string) {
272 ch, reqs, err := newch.Accept()
274 gw.Log.Printf("accept session channel: %s", err)
277 var pty0, tty0 *os.File
278 // Where to send errors/messages for the client to see
279 logw := io.Writer(ch.Stderr())
280 // How to end lines when sending errors/messages to the client
281 // (changes to \r\n when using a pty)
283 // Env vars to add to child process
284 termEnv := []string(nil)
285 for req := range reqs {
288 case "shell", "exec":
293 ssh.Unmarshal(req.Payload, &payload)
294 execargs, err := shlex.Split(payload.Command)
296 fmt.Fprintf(logw, "error parsing supplied command: %s"+eol, err)
299 if len(execargs) == 0 {
300 execargs = []string{"/bin/bash", "-login"}
307 ch.SendRequest("exit-status", false, ssh.Marshal(&resp))
311 cmd, err := gw.Target.InjectCommand(ctx, detachKeys, username, tty0 != nil, execargs)
313 fmt.Fprintln(ch.Stderr(), err)
320 cmd.Stderr = ch.Stderr()
325 var wg sync.WaitGroup
328 go func() { io.Copy(ch, pty0); wg.Done() }()
329 go func() { io.Copy(pty0, ch); wg.Done() }()
330 // Send our own debug messages to tty as well.
333 cmd.SysProcAttr = &syscall.SysProcAttr{
334 Setctty: tty0 != nil,
337 cmd.Env = append(os.Environ(), termEnv...)
339 if exiterr, ok := err.(*exec.ExitError); ok {
340 if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
341 resp.Status = uint32(status.ExitStatus())
343 } else if err != nil {
344 // Propagate errors like `exec: "docker": executable file not found in $PATH`
345 fmt.Fprintln(ch.Stderr(), err)
347 errClose := ch.CloseWrite()
348 if resp.Status == 0 && (err != nil || errClose != nil) {
354 p, t, err := pty.Open()
356 fmt.Fprintf(ch.Stderr(), "pty failed: %s"+eol, err)
370 ssh.Unmarshal(req.Payload, &payload)
371 termEnv = []string{"TERM=" + payload.Term, "USE_TTY=1"}
372 err = pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
374 fmt.Fprintf(logw, "pty-req: setsize failed: %s"+eol, err)
376 case "window-change":
383 ssh.Unmarshal(req.Payload, &payload)
384 err := pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
386 fmt.Fprintf(logw, "window-change: setsize failed: %s"+eol, err)
391 // TODO: implement "env"
392 // requests by setting env
393 // vars in the docker-exec
394 // command (not docker-exec's
395 // own environment, which
396 // would be a gaping security
399 // fmt.Fprintf(logw, "declining %q req"+eol, req.Type)