1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
22 "git.arvados.org/arvados.git/lib/selfsigned"
23 "git.arvados.org/arvados.git/sdk/go/ctxlog"
24 "git.arvados.org/arvados.git/sdk/go/httpserver"
25 "github.com/creack/pty"
26 "github.com/google/shlex"
27 "golang.org/x/crypto/ssh"
28 "golang.org/x/net/context"
31 type GatewayTarget interface {
32 // Command that will execute cmd inside the container
33 InjectCommand(ctx context.Context, detachKeys, username string, usingTTY bool, cmd []string) (*exec.Cmd, error)
35 // IP address inside container
36 IPAddress() (string, error)
39 type GatewayTargetStub struct{}
41 func (GatewayTargetStub) IPAddress() (string, error) { return "127.0.0.1", nil }
42 func (GatewayTargetStub) InjectCommand(ctx context.Context, detachKeys, username string, usingTTY bool, cmd []string) (*exec.Cmd, error) {
43 return exec.CommandContext(ctx, cmd[0], cmd[1:]...), nil
48 Address string // listen host:port; if port=0, Start() will change it to the selected port
52 Printf(fmt string, args ...interface{})
55 sshConfig ssh.ServerConfig
60 // Start starts an http server that allows authenticated clients to open an
61 // interactive "docker exec" session and (in future) connect to tcp ports
62 // inside the docker container.
63 func (gw *Gateway) Start() error {
64 gw.sshConfig = ssh.ServerConfig{
66 PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
70 return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
72 PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
74 return &ssh.Permissions{
75 Extensions: map[string]string{
76 "pubkey-fp": ssh.FingerprintSHA256(pubKey),
80 return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
83 pvt, err := rsa.GenerateKey(rand.Reader, 2048)
91 signer, err := ssh.NewSignerFromKey(pvt)
95 gw.sshConfig.AddHostKey(signer)
97 // Address (typically provided by arvados-dispatch-cloud) is
98 // HOST:PORT where HOST is our IP address or hostname as seen
99 // from arvados-controller, and PORT is either the desired
100 // port where we should run our gateway server, or "0" if we
101 // should choose an available port.
102 host, port, err := net.SplitHostPort(gw.Address)
106 cert, err := selfsigned.CertGenerator{}.Generate()
110 h := hmac.New(sha256.New, []byte(gw.AuthSecret))
111 h.Write(cert.Certificate[0])
112 gw.requestAuth = fmt.Sprintf("%x", h.Sum(nil))
114 h.Write([]byte(gw.requestAuth))
115 gw.respondAuth = fmt.Sprintf("%x", h.Sum(nil))
117 srv := &httpserver.Server{
119 Handler: http.HandlerFunc(gw.handleSSH),
120 TLSConfig: &tls.Config{
121 Certificates: []tls.Certificate{cert},
130 // Get the port number we are listening on (the port might be
131 // "0" or a port name, in which case this will be different).
132 _, port, err = net.SplitHostPort(srv.Addr)
136 // When changing state to Running, we will set
137 // gateway_address to "HOST:PORT" where HOST is our
138 // external hostname/IP as provided by arvados-dispatch-cloud,
139 // and PORT is the port number we ended up listening on.
140 gw.Address = net.JoinHostPort(host, port)
144 // handleSSH connects to an SSH server that allows the caller to run
145 // interactive commands as root (or any other desired user) inside the
146 // container. The tunnel itself can only be created by an
147 // authenticated caller, so the SSH server itself is wide open (any
148 // password or key will be accepted).
150 // Requests must have path "/ssh" and the following headers:
152 // Connection: upgrade
154 // X-Arvados-Target-Uuid: uuid of container
155 // X-Arvados-Authorization: must match
156 // hmac(AuthSecret,certfingerprint) (this prevents other containers
157 // and shell nodes from connecting directly)
161 // X-Arvados-Detach-Keys: argument to "docker exec --detach-keys",
162 // e.g., "ctrl-p,ctrl-q"
163 // X-Arvados-Login-Username: argument to "docker exec --user": account
164 // used to run command(s) inside the container.
165 func (gw *Gateway) handleSSH(w http.ResponseWriter, req *http.Request) {
166 // In future we'll handle browser traffic too, but for now the
167 // only traffic we expect is an SSH tunnel from
168 // (*lib/controller/localdb.Conn)ContainerSSH()
169 if req.Method != "GET" || req.Header.Get("Upgrade") != "ssh" {
170 http.Error(w, "path not found", http.StatusNotFound)
173 if want := req.Header.Get("X-Arvados-Target-Uuid"); want != gw.ContainerUUID {
174 http.Error(w, fmt.Sprintf("misdirected request: meant for %q but received by crunch-run %q", want, gw.ContainerUUID), http.StatusBadGateway)
177 if req.Header.Get("X-Arvados-Authorization") != gw.requestAuth {
178 http.Error(w, "bad X-Arvados-Authorization header", http.StatusUnauthorized)
181 detachKeys := req.Header.Get("X-Arvados-Detach-Keys")
182 username := req.Header.Get("X-Arvados-Login-Username")
186 hj, ok := w.(http.Hijacker)
188 http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
191 netconn, _, err := hj.Hijack()
193 http.Error(w, err.Error(), http.StatusInternalServerError)
196 defer netconn.Close()
197 w.Header().Set("Connection", "upgrade")
198 w.Header().Set("Upgrade", "ssh")
199 w.Header().Set("X-Arvados-Authorization-Response", gw.respondAuth)
200 netconn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\n"))
201 w.Header().Write(netconn)
202 netconn.Write([]byte("\r\n"))
206 conn, newchans, reqs, err := ssh.NewServerConn(netconn, &gw.sshConfig)
208 gw.Log.Printf("ssh.NewServerConn: %s", err)
212 go ssh.DiscardRequests(reqs)
213 for newch := range newchans {
214 switch newch.ChannelType() {
216 go gw.handleDirectTCPIP(ctx, newch)
218 go gw.handleSession(ctx, newch, detachKeys, username)
220 go newch.Reject(ssh.UnknownChannelType, fmt.Sprintf("unsupported channel type %q", newch.ChannelType()))
225 func (gw *Gateway) handleDirectTCPIP(ctx context.Context, newch ssh.NewChannel) {
226 ch, reqs, err := newch.Accept()
228 gw.Log.Printf("accept direct-tcpip channel: %s", err)
232 go ssh.DiscardRequests(reqs)
234 // RFC 4254 7.2 (copy of channelOpenDirectMsg in
235 // golang.org/x/crypto/ssh)
242 err = ssh.Unmarshal(newch.ExtraData(), &msg)
244 fmt.Fprintf(ch.Stderr(), "unmarshal direct-tcpip extradata: %s\n", err)
248 case "localhost", "0.0.0.0", "127.0.0.1", "::1", "::":
250 fmt.Fprintf(ch.Stderr(), "cannot forward to ports on %q, only localhost\n", msg.Raddr)
254 dstaddr, err := gw.Target.IPAddress()
256 fmt.Fprintf(ch.Stderr(), "container has no IP address: %s\n", err)
258 } else if dstaddr == "" {
259 fmt.Fprintf(ch.Stderr(), "container has no IP address\n")
263 dst := net.JoinHostPort(dstaddr, fmt.Sprintf("%d", msg.Rport))
264 tcpconn, err := net.Dial("tcp", dst)
266 fmt.Fprintf(ch.Stderr(), "%s: %s\n", dst, err)
270 n, _ := io.Copy(ch, tcpconn)
271 ctxlog.FromContext(ctx).Debugf("tcpip: sent %d bytes\n", n)
274 n, _ := io.Copy(tcpconn, ch)
275 ctxlog.FromContext(ctx).Debugf("tcpip: received %d bytes\n", n)
278 func (gw *Gateway) handleSession(ctx context.Context, newch ssh.NewChannel, detachKeys, username string) {
279 ch, reqs, err := newch.Accept()
281 gw.Log.Printf("accept session channel: %s", err)
284 var pty0, tty0 *os.File
285 // Where to send errors/messages for the client to see
286 logw := io.Writer(ch.Stderr())
287 // How to end lines when sending errors/messages to the client
288 // (changes to \r\n when using a pty)
290 // Env vars to add to child process
291 termEnv := []string(nil)
292 for req := range reqs {
295 case "shell", "exec":
300 ssh.Unmarshal(req.Payload, &payload)
301 execargs, err := shlex.Split(payload.Command)
303 fmt.Fprintf(logw, "error parsing supplied command: %s"+eol, err)
306 if len(execargs) == 0 {
307 execargs = []string{"/bin/bash", "-login"}
314 ch.SendRequest("exit-status", false, ssh.Marshal(&resp))
318 cmd, err := gw.Target.InjectCommand(ctx, detachKeys, username, tty0 != nil, execargs)
320 fmt.Fprintln(ch.Stderr(), err)
327 cmd.Stderr = ch.Stderr()
332 var wg sync.WaitGroup
335 go func() { io.Copy(ch, pty0); wg.Done() }()
336 go func() { io.Copy(pty0, ch); wg.Done() }()
337 // Send our own debug messages to tty as well.
340 cmd.SysProcAttr = &syscall.SysProcAttr{
341 Setctty: tty0 != nil,
344 cmd.Env = append(os.Environ(), termEnv...)
346 if exiterr, ok := err.(*exec.ExitError); ok {
347 if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
348 resp.Status = uint32(status.ExitStatus())
350 } else if err != nil {
351 // Propagate errors like `exec: "docker": executable file not found in $PATH`
352 fmt.Fprintln(ch.Stderr(), err)
354 errClose := ch.CloseWrite()
355 if resp.Status == 0 && (err != nil || errClose != nil) {
361 p, t, err := pty.Open()
363 fmt.Fprintf(ch.Stderr(), "pty failed: %s"+eol, err)
377 ssh.Unmarshal(req.Payload, &payload)
378 termEnv = []string{"TERM=" + payload.Term, "USE_TTY=1"}
379 err = pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
381 fmt.Fprintf(logw, "pty-req: setsize failed: %s"+eol, err)
383 case "window-change":
390 ssh.Unmarshal(req.Payload, &payload)
391 err := pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
393 fmt.Fprintf(logw, "window-change: setsize failed: %s"+eol, err)
398 // TODO: implement "env"
399 // requests by setting env
400 // vars in the docker-exec
401 // command (not docker-exec's
402 // own environment, which
403 // would be a gaping security
406 // fmt.Fprintf(logw, "declining %q req"+eol, req.Type)