1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
26 "git.arvados.org/arvados.git/lib/controller/rpc"
27 "git.arvados.org/arvados.git/lib/selfsigned"
28 "git.arvados.org/arvados.git/lib/webdavfs"
29 "git.arvados.org/arvados.git/sdk/go/arvados"
30 "git.arvados.org/arvados.git/sdk/go/auth"
31 "git.arvados.org/arvados.git/sdk/go/ctxlog"
32 "git.arvados.org/arvados.git/sdk/go/httpserver"
33 "github.com/creack/pty"
34 "github.com/google/shlex"
35 "github.com/hashicorp/yamux"
36 "golang.org/x/crypto/ssh"
37 "golang.org/x/net/webdav"
40 type GatewayTarget interface {
41 // Command that will execute cmd inside the container
42 InjectCommand(ctx context.Context, detachKeys, username string, usingTTY bool, cmd []string) (*exec.Cmd, error)
44 // IP address inside container
45 IPAddress() (string, error)
48 type GatewayTargetStub struct{}
50 func (GatewayTargetStub) IPAddress() (string, error) { return "127.0.0.1", nil }
51 func (GatewayTargetStub) InjectCommand(ctx context.Context, detachKeys, username string, usingTTY bool, cmd []string) (*exec.Cmd, error) {
52 return exec.CommandContext(ctx, cmd[0], cmd[1:]...), nil
57 // Caller should set Address to "", or "host:0" or "host:port"
58 // where host is a known external IP address; port is a
59 // desired port number to listen on; and ":0" chooses an
60 // available dynamic port.
62 // If Address is "", Start() listens only on the loopback
63 // interface (and changes Address to "127.0.0.1:port").
64 // Otherwise it listens on all interfaces.
66 // If Address is "host:0", Start() updates Address to
72 Printf(fmt string, args ...interface{})
74 // If non-nil, set up a ContainerGatewayTunnel, so that the
75 // controller can connect to us even if our external IP
76 // address is unknown or not routable from controller.
77 ArvadosClient *arvados.Client
79 // When a tunnel is connected or reconnected, this func (if
80 // not nil) will be called with the InternalURL of the
81 // controller process at the other end of the tunnel.
82 UpdateTunnelURL func(url string)
84 // Source for serving WebDAV requests with
85 // X-Webdav-Source: /log
86 LogCollection arvados.CollectionFileSystem
88 sshConfig ssh.ServerConfig
93 // Start starts an http server that allows authenticated clients to open an
94 // interactive "docker exec" session and (in future) connect to tcp ports
95 // inside the docker container.
96 func (gw *Gateway) Start() error {
97 gw.sshConfig = ssh.ServerConfig{
99 PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
103 return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
105 PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
107 return &ssh.Permissions{
108 Extensions: map[string]string{
109 "pubkey-fp": ssh.FingerprintSHA256(pubKey),
113 return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
116 pvt, err := rsa.GenerateKey(rand.Reader, 2048)
124 signer, err := ssh.NewSignerFromKey(pvt)
128 gw.sshConfig.AddHostKey(signer)
130 // Address (typically provided by arvados-dispatch-cloud) is
131 // HOST:PORT where HOST is our IP address or hostname as seen
132 // from arvados-controller, and PORT is either the desired
133 // port where we should run our gateway server, or "0" if we
134 // should choose an available port.
135 extAddr := gw.Address
136 // Generally we can't know which local interface corresponds
137 // to an externally reachable IP address, so if we expect to
138 // be reachable by external hosts, we listen on all
142 // If the dispatcher doesn't tell us our external IP
143 // address, controller will only be able to connect
144 // through the tunnel (see runTunnel), so our gateway
145 // server only needs to listen on the loopback
147 extAddr = "127.0.0.1:0"
148 listenHost = "127.0.0.1"
150 extHost, extPort, err := net.SplitHostPort(extAddr)
154 cert, err := selfsigned.CertGenerator{}.Generate()
158 h := hmac.New(sha256.New, []byte(gw.AuthSecret))
159 h.Write(cert.Certificate[0])
160 gw.requestAuth = fmt.Sprintf("%x", h.Sum(nil))
162 h.Write([]byte(gw.requestAuth))
163 gw.respondAuth = fmt.Sprintf("%x", h.Sum(nil))
165 srv := &httpserver.Server{
168 TLSConfig: &tls.Config{
169 Certificates: []tls.Certificate{cert},
172 Addr: net.JoinHostPort(listenHost, extPort),
180 gw.Log.Printf("gateway server stopped: %s", err)
182 // Get the port number we are listening on (extPort might be
183 // "0" or a port name, in which case this will be different).
184 _, listenPort, err := net.SplitHostPort(srv.Addr)
188 // When changing state to Running, the caller will want to set
189 // gateway_address to a "HOST:PORT" that, if controller
190 // connects to it, will reach this gateway server.
192 // The most likely thing to work is: HOST is our external
193 // hostname/IP as provided by the caller
194 // (arvados-dispatch-cloud) or 127.0.0.1 to indicate
195 // non-tunnel connections aren't available; and PORT is the
196 // port number we are listening on.
197 gw.Address = net.JoinHostPort(extHost, listenPort)
198 gw.Log.Printf("gateway server listening at %s", gw.Address)
199 if gw.ArvadosClient != nil {
200 go gw.maintainTunnel(gw.Address)
205 func (gw *Gateway) maintainTunnel(addr string) {
206 for ; ; time.Sleep(5 * time.Second) {
207 err := gw.runTunnel(addr)
208 gw.Log.Printf("runTunnel: %s", err)
212 // runTunnel connects to controller and sets up a tunnel through
213 // which controller can connect to the gateway server at the given
215 func (gw *Gateway) runTunnel(addr string) error {
216 ctx := auth.NewContext(context.Background(), auth.NewCredentials(gw.ArvadosClient.AuthToken))
217 arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: gw.ArvadosClient.APIHost}, gw.ArvadosClient.Insecure, rpc.PassthroughTokenProvider)
218 tun, err := arpc.ContainerGatewayTunnel(ctx, arvados.ContainerGatewayTunnelOptions{
219 UUID: gw.ContainerUUID,
220 AuthSecret: gw.AuthSecret,
223 return fmt.Errorf("error creating gateway tunnel: %w", err)
225 mux, err := yamux.Client(tun.Conn, nil)
227 return fmt.Errorf("error setting up mux client end: %s", err)
229 if url := tun.Header.Get("X-Arvados-Internal-Url"); url != "" && gw.UpdateTunnelURL != nil {
230 gw.UpdateTunnelURL(url)
233 muxconn, err := mux.AcceptStream()
237 gw.Log.Printf("tunnel connection %d started", muxconn.StreamID())
239 defer muxconn.Close()
240 gwconn, err := net.Dial("tcp", addr)
242 gw.Log.Printf("tunnel connection %d: error connecting to %s: %s", muxconn.StreamID(), addr, err)
246 var wg sync.WaitGroup
250 _, err := io.Copy(gwconn, muxconn)
252 gw.Log.Printf("tunnel connection %d: mux end: %s", muxconn.StreamID(), err)
258 _, err := io.Copy(muxconn, gwconn)
260 gw.Log.Printf("tunnel connection %d: gateway end: %s", muxconn.StreamID(), err)
265 gw.Log.Printf("tunnel connection %d finished", muxconn.StreamID())
270 var webdavMethod = map[string]bool{
276 func (gw *Gateway) ServeHTTP(w http.ResponseWriter, req *http.Request) {
277 w.Header().Set("Vary", "X-Arvados-Authorization, X-Arvados-Container-Gateway-Uuid, X-Webdav-Prefix, X-Webdav-Source")
278 reqUUID := req.Header.Get("X-Arvados-Container-Gateway-Uuid")
280 // older controller versions only send UUID as query param
282 reqUUID = req.Form.Get("uuid")
284 if reqUUID != gw.ContainerUUID {
285 http.Error(w, fmt.Sprintf("misdirected request: meant for %q but received by crunch-run %q", reqUUID, gw.ContainerUUID), http.StatusBadGateway)
288 if req.Header.Get("X-Arvados-Authorization") != gw.requestAuth {
289 http.Error(w, "bad X-Arvados-Authorization header", http.StatusUnauthorized)
292 w.Header().Set("X-Arvados-Authorization-Response", gw.respondAuth)
294 case req.Method == "POST" && req.Header.Get("Upgrade") == "ssh":
296 case req.Header.Get("X-Webdav-Source") == "/log":
297 if !webdavMethod[req.Method] {
298 http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
301 gw.handleLogsWebDAV(w, req)
303 http.Error(w, "path not found", http.StatusNotFound)
307 func (gw *Gateway) handleLogsWebDAV(w http.ResponseWriter, r *http.Request) {
308 prefix := r.Header.Get("X-Webdav-Prefix")
309 if !strings.HasPrefix(r.URL.Path, prefix) {
310 http.Error(w, "X-Webdav-Prefix header is not a prefix of the requested path", http.StatusBadRequest)
313 if gw.LogCollection == nil {
314 http.Error(w, "Not found", http.StatusNotFound)
317 wh := webdav.Handler{
319 FileSystem: &webdavfs.FS{
320 FileSystem: gw.LogCollection,
323 AlwaysReadEOF: r.Method == "PROPFIND",
325 LockSystem: webdavfs.NoLockSystem,
326 Logger: gw.webdavLogger,
331 func (gw *Gateway) webdavLogger(r *http.Request, err error) {
332 if err != nil && !os.IsNotExist(err) {
333 ctxlog.FromContext(r.Context()).WithError(err).Info("error reported by webdav handler")
335 ctxlog.FromContext(r.Context()).WithError(err).Debug("webdav request log")
339 // handleSSH connects to an SSH server that allows the caller to run
340 // interactive commands as root (or any other desired user) inside the
341 // container. The tunnel itself can only be created by an
342 // authenticated caller, so the SSH server itself is wide open (any
343 // password or key will be accepted).
345 // Requests must have path "/ssh" and the following headers:
347 // Connection: upgrade
349 // X-Arvados-Target-Uuid: uuid of container
350 // X-Arvados-Authorization: must match
351 // hmac(AuthSecret,certfingerprint) (this prevents other containers
352 // and shell nodes from connecting directly)
356 // X-Arvados-Detach-Keys: argument to "docker exec --detach-keys",
357 // e.g., "ctrl-p,ctrl-q"
358 // X-Arvados-Login-Username: argument to "docker exec --user": account
359 // used to run command(s) inside the container.
360 func (gw *Gateway) handleSSH(w http.ResponseWriter, req *http.Request) {
362 detachKeys := req.Form.Get("detach_keys")
363 username := req.Form.Get("login_username")
367 hj, ok := w.(http.Hijacker)
369 http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
372 netconn, _, err := hj.Hijack()
374 http.Error(w, err.Error(), http.StatusInternalServerError)
377 defer netconn.Close()
378 w.Header().Set("Connection", "upgrade")
379 w.Header().Set("Upgrade", "ssh")
380 netconn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\n"))
381 w.Header().Write(netconn)
382 netconn.Write([]byte("\r\n"))
386 conn, newchans, reqs, err := ssh.NewServerConn(netconn, &gw.sshConfig)
389 } else if err != nil {
390 gw.Log.Printf("ssh.NewServerConn: %s", err)
394 go ssh.DiscardRequests(reqs)
395 for newch := range newchans {
396 switch newch.ChannelType() {
398 go gw.handleDirectTCPIP(ctx, newch)
400 go gw.handleSession(ctx, newch, detachKeys, username)
402 go newch.Reject(ssh.UnknownChannelType, fmt.Sprintf("unsupported channel type %q", newch.ChannelType()))
407 func (gw *Gateway) handleDirectTCPIP(ctx context.Context, newch ssh.NewChannel) {
408 ch, reqs, err := newch.Accept()
410 gw.Log.Printf("accept direct-tcpip channel: %s", err)
414 go ssh.DiscardRequests(reqs)
416 // RFC 4254 7.2 (copy of channelOpenDirectMsg in
417 // golang.org/x/crypto/ssh)
424 err = ssh.Unmarshal(newch.ExtraData(), &msg)
426 fmt.Fprintf(ch.Stderr(), "unmarshal direct-tcpip extradata: %s\n", err)
430 case "localhost", "0.0.0.0", "127.0.0.1", "::1", "::":
432 fmt.Fprintf(ch.Stderr(), "cannot forward to ports on %q, only localhost\n", msg.Raddr)
436 dstaddr, err := gw.Target.IPAddress()
438 fmt.Fprintf(ch.Stderr(), "container has no IP address: %s\n", err)
440 } else if dstaddr == "" {
441 fmt.Fprintf(ch.Stderr(), "container has no IP address\n")
445 dst := net.JoinHostPort(dstaddr, fmt.Sprintf("%d", msg.Rport))
446 tcpconn, err := net.Dial("tcp", dst)
448 fmt.Fprintf(ch.Stderr(), "%s: %s\n", dst, err)
452 n, _ := io.Copy(ch, tcpconn)
453 ctxlog.FromContext(ctx).Debugf("tcpip: sent %d bytes\n", n)
456 n, _ := io.Copy(tcpconn, ch)
457 ctxlog.FromContext(ctx).Debugf("tcpip: received %d bytes\n", n)
460 func (gw *Gateway) handleSession(ctx context.Context, newch ssh.NewChannel, detachKeys, username string) {
461 ch, reqs, err := newch.Accept()
463 gw.Log.Printf("error accepting session channel: %s", err)
468 var pty0, tty0 *os.File
469 // Where to send errors/messages for the client to see
470 logw := io.Writer(ch.Stderr())
471 // How to end lines when sending errors/messages to the client
472 // (changes to \r\n when using a pty)
474 // Env vars to add to child process
475 termEnv := []string(nil)
478 wantClose := make(chan struct{})
482 case r, ok := <-reqs:
492 case "shell", "exec":
493 if started++; started != 1 {
494 // RFC 4254 6.5: "Only one of these
495 // requests can succeed per channel."
502 ssh.Unmarshal(req.Payload, &payload)
503 execargs, err := shlex.Split(payload.Command)
505 fmt.Fprintf(logw, "error parsing supplied command: %s"+eol, err)
508 if len(execargs) == 0 {
509 execargs = []string{"/bin/bash", "-login"}
516 ch.SendRequest("exit-status", false, ssh.Marshal(&resp))
520 cmd, err := gw.Target.InjectCommand(ctx, detachKeys, username, tty0 != nil, execargs)
522 fmt.Fprintln(ch.Stderr(), err)
533 // Send our own debug messages to tty as well.
536 // StdinPipe may seem
537 // superfluous here, but it's
538 // not: it causes cmd.Run() to
539 // return when the subprocess
540 // exits. Without it, Run()
541 // waits for stdin to close,
542 // which causes "ssh ... echo
543 // ok" (with the client's
544 // stdin connected to a
545 // terminal or something) to
547 stdin, err := cmd.StdinPipe()
549 fmt.Fprintln(ch.Stderr(), err)
559 cmd.Stderr = ch.Stderr()
561 cmd.SysProcAttr = &syscall.SysProcAttr{
562 Setctty: tty0 != nil,
565 cmd.Env = append(os.Environ(), termEnv...)
567 if exiterr, ok := err.(*exec.ExitError); ok {
568 if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
569 resp.Status = uint32(status.ExitStatus())
571 } else if err != nil {
572 // Propagate errors like `exec: "docker": executable file not found in $PATH`
573 fmt.Fprintln(ch.Stderr(), err)
575 errClose := ch.CloseWrite()
576 if resp.Status == 0 && (err != nil || errClose != nil) {
582 p, t, err := pty.Open()
584 fmt.Fprintf(ch.Stderr(), "pty failed: %s"+eol, err)
598 ssh.Unmarshal(req.Payload, &payload)
599 termEnv = []string{"TERM=" + payload.Term, "USE_TTY=1"}
600 err = pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
602 fmt.Fprintf(logw, "pty-req: setsize failed: %s"+eol, err)
604 case "window-change":
611 ssh.Unmarshal(req.Payload, &payload)
612 err := pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
614 fmt.Fprintf(logw, "window-change: setsize failed: %s"+eol, err)
619 // TODO: implement "env"
620 // requests by setting env
621 // vars in the docker-exec
622 // command (not docker-exec's
623 // own environment, which
624 // would be a gaping security
627 // fmt.Fprintf(logw, "declined request %q on ssh channel"+eol, req.Type)