From: Tom Clegg <tom@curii.com>
Date: Wed, 3 Feb 2021 16:46:43 +0000 (-0500)
Subject: 17170: Merge branch 'master'
X-Git-Tag: 2.2.0~135^2~2
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/38a0f5e77f2190487d03d6538337d3b7055fd1e8?hp=08c80b2ba6a8fc090d2e998dfcf2f280c8508019

17170: Merge branch 'master'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
---

diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index 47fcd5ad7d..aefcce79a4 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -57,6 +57,8 @@ var (
 		"mount":                mount.Command,
 		"deduplication-report": deduplicationreport.Command,
 		"costanalyzer":         costanalyzer.Command,
+		"shell":                shellCommand{},
+		"connect-ssh":          connectSSHCommand{},
 	})
 )
 
diff --git a/cmd/arvados-client/container_gateway.go b/cmd/arvados-client/container_gateway.go
new file mode 100644
index 0000000000..6dbc241bca
--- /dev/null
+++ b/cmd/arvados-client/container_gateway.go
@@ -0,0 +1,202 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+
+	"git.arvados.org/arvados.git/lib/controller/rpc"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// shellCommand connects the terminal to an interactive shell on a
+// running container.
+type shellCommand struct{}
+
+func (shellCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	f := flag.NewFlagSet(prog, flag.ContinueOnError)
+	f.SetOutput(stderr)
+	f.Usage = func() {
+		fmt.Fprint(stderr, prog+`: open an interactive shell on a running container.
+
+Usage: `+prog+` [options] [username@]container-uuid [ssh-options] [remote-command [args...]]
+
+Options:
+`)
+		f.PrintDefaults()
+	}
+	detachKeys := f.String("detach-keys", "ctrl-],ctrl-]", "set detach key sequence, as in docker-attach(1)")
+	err := f.Parse(args)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+
+	if f.NArg() < 1 {
+		f.Usage()
+		return 2
+	}
+	target := f.Args()[0]
+	if !strings.Contains(target, "@") {
+		target = "root@" + target
+	}
+	sshargs := f.Args()[1:]
+
+	// Try setting up a tunnel, and exit right away if it
+	// fails. This tunnel won't get used -- we'll set up a new
+	// tunnel when running as SSH client's ProxyCommand child --
+	// but in most cases where the real tunnel setup would fail,
+	// we catch the problem earlier here. This makes it less
+	// likely that an error message about tunnel setup will get
+	// hidden behind noisy errors from SSH client like this:
+	//
+	// [useful tunnel setup error message here]
+	// kex_exchange_identification: Connection closed by remote host
+	// Connection closed by UNKNOWN port 65535
+	// exit status 255
+	exitcode := connectSSHCommand{}.RunCommand(
+		"arvados-client connect-ssh",
+		[]string{"-detach-keys=" + *detachKeys, "-probe-only=true", target},
+		&bytes.Buffer{}, &bytes.Buffer{}, stderr)
+	if exitcode != 0 {
+		return exitcode
+	}
+
+	selfbin, err := os.Readlink("/proc/self/exe")
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	}
+	sshargs = append([]string{
+		"ssh",
+		"-o", "ProxyCommand " + selfbin + " connect-ssh -detach-keys=" + shellescape(*detachKeys) + " " + shellescape(target),
+		"-o", "StrictHostKeyChecking no",
+		target},
+		sshargs...)
+	sshbin, err := exec.LookPath("ssh")
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	err = syscall.Exec(sshbin, sshargs, os.Environ())
+	fmt.Fprintf(stderr, "exec(%q) failed: %s\n", sshbin, err)
+	return 1
+}
+
+// connectSSHCommand connects stdin/stdout to a container's gateway
+// server (see lib/crunchrun/ssh.go).
+//
+// It is intended to be invoked with OpenSSH client's ProxyCommand
+// config.
+type connectSSHCommand struct{}
+
+func (connectSSHCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	f := flag.NewFlagSet(prog, flag.ContinueOnError)
+	f.SetOutput(stderr)
+	f.Usage = func() {
+		fmt.Fprint(stderr, prog+`: connect to the gateway service for a running container.
+
+Usage: `+prog+` [options] [username@]container-uuid
+
+Options:
+`)
+		f.PrintDefaults()
+	}
+	probeOnly := f.Bool("probe-only", false, "do not transfer IO, just exit 0 immediately if tunnel setup succeeds")
+	detachKeys := f.String("detach-keys", "", "set detach key sequence, as in docker-attach(1)")
+	if err := f.Parse(args); err != nil {
+		fmt.Fprintln(stderr, err)
+		return 2
+	} else if f.NArg() != 1 {
+		f.Usage()
+		return 2
+	}
+	targetUUID := f.Args()[0]
+	loginUsername := "root"
+	if i := strings.Index(targetUUID, "@"); i >= 0 {
+		loginUsername = targetUUID[:i]
+		targetUUID = targetUUID[i+1:]
+	}
+	if os.Getenv("ARVADOS_API_HOST") == "" || os.Getenv("ARVADOS_API_TOKEN") == "" {
+		fmt.Fprintln(stderr, "fatal: ARVADOS_API_HOST and ARVADOS_API_TOKEN environment variables are not set")
+		return 1
+	}
+	insecure := os.Getenv("ARVADOS_API_HOST_INSECURE")
+	rpcconn := rpc.NewConn("",
+		&url.URL{
+			Scheme: "https",
+			Host:   os.Getenv("ARVADOS_API_HOST"),
+		},
+		insecure == "1" || insecure == "yes" || insecure == "true",
+		func(context.Context) ([]string, error) {
+			return []string{os.Getenv("ARVADOS_API_TOKEN")}, nil
+		})
+	if strings.Contains(targetUUID, "-xvhdp-") {
+		crs, err := rpcconn.ContainerRequestList(context.TODO(), arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"uuid", "=", targetUUID}}})
+		if err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		if len(crs.Items) < 1 {
+			fmt.Fprintf(stderr, "container request %q not found\n", targetUUID)
+			return 1
+		}
+		cr := crs.Items[0]
+		if cr.ContainerUUID == "" {
+			fmt.Fprintf(stderr, "no container assigned, container request state is %s\n", strings.ToLower(string(cr.State)))
+			return 1
+		}
+		targetUUID = cr.ContainerUUID
+		fmt.Fprintln(stderr, "connecting to container", targetUUID)
+	} else if !strings.Contains(targetUUID, "-dz642-") {
+		fmt.Fprintf(stderr, "target UUID is not a container or container request UUID: %s\n", targetUUID)
+		return 1
+	}
+	sshconn, err := rpcconn.ContainerSSH(context.TODO(), arvados.ContainerSSHOptions{
+		UUID:          targetUUID,
+		DetachKeys:    *detachKeys,
+		LoginUsername: loginUsername,
+	})
+	if err != nil {
+		fmt.Fprintln(stderr, "error setting up tunnel:", err)
+		return 1
+	}
+	defer sshconn.Conn.Close()
+
+	if *probeOnly {
+		return 0
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		defer cancel()
+		_, err := io.Copy(stdout, sshconn.Conn)
+		if err != nil && ctx.Err() == nil {
+			fmt.Fprintf(stderr, "receive: %v\n", err)
+		}
+	}()
+	go func() {
+		defer cancel()
+		_, err := io.Copy(sshconn.Conn, stdin)
+		if err != nil && ctx.Err() == nil {
+			fmt.Fprintf(stderr, "send: %v\n", err)
+		}
+	}()
+	<-ctx.Done()
+	return 0
+}
+
+func shellescape(s string) string {
+	return "'" + strings.Replace(s, "'", "'\\''", -1) + "'"
+}
diff --git a/cmd/arvados-client/container_gateway_test.go b/cmd/arvados-client/container_gateway_test.go
new file mode 100644
index 0000000000..39d6e910b8
--- /dev/null
+++ b/cmd/arvados-client/container_gateway_test.go
@@ -0,0 +1,82 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"fmt"
+	"net/url"
+	"os"
+	"os/exec"
+
+	"git.arvados.org/arvados.git/lib/controller/rpc"
+	"git.arvados.org/arvados.git/lib/crunchrun"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	check "gopkg.in/check.v1"
+)
+
+func (s *ClientSuite) TestShellGatewayNotAvailable(c *check.C) {
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command("go", "run", ".", "shell", arvadostest.QueuedContainerUUID, "-o", "controlpath=none", "echo", "ok")
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	c.Check(cmd.Run(), check.NotNil)
+	c.Log(stderr.String())
+	c.Check(stderr.String(), check.Matches, `(?ms).*gateway is not available, container is queued.*`)
+}
+
+func (s *ClientSuite) TestShellGateway(c *check.C) {
+	defer func() {
+		c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+	}()
+	uuid := arvadostest.QueuedContainerUUID
+	h := hmac.New(sha256.New, []byte(arvadostest.SystemRootToken))
+	fmt.Fprint(h, uuid)
+	authSecret := fmt.Sprintf("%x", h.Sum(nil))
+	dcid := "theperthcountyconspiracy"
+	gw := crunchrun.Gateway{
+		DockerContainerID: &dcid,
+		ContainerUUID:     uuid,
+		Address:           "0.0.0.0:0",
+		AuthSecret:        authSecret,
+	}
+	err := gw.Start()
+	c.Assert(err, check.IsNil)
+
+	rpcconn := rpc.NewConn("",
+		&url.URL{
+			Scheme: "https",
+			Host:   os.Getenv("ARVADOS_API_HOST"),
+		},
+		true,
+		func(context.Context) ([]string, error) {
+			return []string{arvadostest.SystemRootToken}, nil
+		})
+	_, err = rpcconn.ContainerUpdate(context.TODO(), arvados.UpdateOptions{UUID: uuid, Attrs: map[string]interface{}{
+		"state": arvados.ContainerStateLocked,
+	}})
+	c.Assert(err, check.IsNil)
+	_, err = rpcconn.ContainerUpdate(context.TODO(), arvados.UpdateOptions{UUID: uuid, Attrs: map[string]interface{}{
+		"state":           arvados.ContainerStateRunning,
+		"gateway_address": gw.Address,
+	}})
+	c.Assert(err, check.IsNil)
+
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command("go", "run", ".", "shell", uuid, "-o", "controlpath=none", "-o", "userknownhostsfile="+c.MkDir()+"/known_hosts", "echo", "ok")
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	cmd.Env = append(cmd.Env, "ARVADOS_API_TOKEN="+arvadostest.ActiveTokenV2)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	c.Check(cmd.Run(), check.NotNil)
+	c.Log(stderr.String())
+	c.Check(stderr.String(), check.Matches, `(?ms).*(No such container: theperthcountyconspiracy|exec: \"docker\": executable file not found in \$PATH).*`)
+}
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 8317a744a7..e0324f6b86 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -39,6 +39,15 @@ h2(#main). development main (as of 2020-12-10)
 
 "Upgrading from 2.1.0":#v2_1_0
 
+h3. New proxy parameters for arvados-controller
+
+In your Nginx configuration file (@/etc/nginx/conf.d/arvados-api-and-controller.conf@), add the following lines to the @location /@ block with @http://controller@ (see "Update nginx configuration":{{site.baseurl}}/install/install-api-server.html#update-nginx for an example) and reload/restart Nginx (@sudo nginx -s reload@).
+
+<pre>
+    proxy_set_header      Upgrade           $http_upgrade;
+    proxy_set_header      Connection        "upgrade";
+</pre>
+
 h3. Changes on the collection's @preserve_version@ attribute semantics
 
 The @preserve_version@ attribute on collections was originally designed to allow clients to persist a preexisting collection version. This forced clients to make 2 requests if the intention is to "make this set of changes in a new version that will be kept", so we have changed the semantics to do just that: When passing @preserve_version=true@ along with other collection updates, the current version is persisted and also the newly created one will be persisted on the next update.
diff --git a/doc/api/methods/containers.html.textile.liquid b/doc/api/methods/containers.html.textile.liquid
index 8a7ebc36e5..096a1fcaa9 100644
--- a/doc/api/methods/containers.html.textile.liquid
+++ b/doc/api/methods/containers.html.textile.liquid
@@ -57,6 +57,8 @@ Generally this will contain additional keys that are not present in any correspo
 |auth_uuid|string|UUID of a token to be passed into the container itself, used to access Keep-backed mounts, etc.  Automatically assigned.|Null if state∉{"Locked","Running"} or if @runtime_token@ was provided.|
 |locked_by_uuid|string|UUID of a token, indicating which dispatch process changed state to Locked. If null, any token can be used to lock. If not null, only the indicated token can modify this container.|Null if state∉{"Locked","Running"}|
 |runtime_token|string|A v2 token to be passed into the container itself, used to access Keep-backed mounts, etc.|Not returned in API responses.  Reset to null when state is "Complete" or "Cancelled".|
+|gateway_address|string|Address (host:port) of gateway server.|Internal use only.|
+|interactive_session_started|boolean|Indicates whether @arvados-client shell@ has been used to run commands in the container, which may have altered the container's behavior and output.||
 
 h2(#container_states). Container states
 
diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid
index ca55be53e3..7d0353c9e7 100644
--- a/doc/install/install-api-server.html.textile.liquid
+++ b/doc/install/install-api-server.html.textile.liquid
@@ -153,11 +153,13 @@ server {
     proxy_connect_timeout 90s;
     proxy_read_timeout    300s;
 
-    proxy_set_header      X-Forwarded-Proto https;
-    proxy_set_header      Host $http_host;
+    proxy_set_header      Host              $http_host;
+    proxy_set_header      Upgrade           $http_upgrade;
+    proxy_set_header      Connection        "upgrade";
     proxy_set_header      X-External-Client $external_client;
-    proxy_set_header      X-Real-IP $remote_addr;
-    proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header      X-Forwarded-For   $proxy_add_x_forwarded_for;
+    proxy_set_header      X-Forwarded-Proto https;
+    proxy_set_header      X-Real-IP         $remote_addr;
   }
 }
 
diff --git a/go.mod b/go.mod
index 262978d912..88dcb86c72 100644
--- a/go.mod
+++ b/go.mod
@@ -20,6 +20,7 @@ require (
 	github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7
+	github.com/creack/pty v1.1.7
 	github.com/dnaeon/go-vcr v1.0.1 // indirect
 	github.com/docker/distribution v2.6.0-rc.1.0.20180105232752-277ed486c948+incompatible // indirect
 	github.com/docker/docker v1.4.2-0.20180109013817-94b8a116fbf1
@@ -33,6 +34,7 @@ require (
 	github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
 	github.com/go-ldap/ldap v3.0.3+incompatible
 	github.com/gogo/protobuf v1.1.1
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/mux v1.6.1-0.20180107155708-5bbbb5b2b572
 	github.com/hashicorp/golang-lru v0.5.1
diff --git a/go.sum b/go.sum
index 85d205112f..91b5689eb3 100644
--- a/go.sum
+++ b/go.sum
@@ -72,6 +72,8 @@ github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7 h1:e3u8KWFMR3irlDo1Z/tL8Hsz1MJmCLkSoX5AZRMKZkg=
 github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -134,6 +136,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 771dc2ee79..f62c483f46 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -868,6 +868,26 @@ Clusters:
         # period.
         LogUpdateSize: 32MiB
 
+      ShellAccess:
+        # An admin user can use "arvados-client shell" to start an
+        # interactive shell (with any user ID) in any running
+        # container.
+        Admin: false
+
+        # Any user can use "arvados-client shell" to start an
+        # interactive shell (with any user ID) in any running
+        # container that they started, provided it isn't also
+        # associated with a different user's container request.
+        #
+        # Interactive sessions make it easy to alter the container's
+        # runtime environment in ways that aren't recorded or
+        # reproducible. Consider the implications for automatic
+        # container reuse before enabling and using this feature. In
+        # particular, note that starting an interactive session does
+        # not disqualify a container from being reused by a different
+        # user/workflow in the future.
+        User: false
+
       SLURM:
         PrioritySpread: 0
         SbatchArgumentsList: []
diff --git a/lib/config/export.go b/lib/config/export.go
index e4917032ff..2d666e638b 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -121,6 +121,9 @@ var whitelist = map[string]bool{
 	"Containers.MaxRetryAttempts":                  true,
 	"Containers.MinRetryPeriod":                    true,
 	"Containers.ReserveExtraRAM":                   true,
+	"Containers.ShellAccess":                       true,
+	"Containers.ShellAccess.Admin":                 true,
+	"Containers.ShellAccess.User":                  true,
 	"Containers.SLURM":                             false,
 	"Containers.StaleLockTimeout":                  false,
 	"Containers.SupportedDockerImageFormats":       true,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index a202a54047..df4b028622 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -874,6 +874,26 @@ Clusters:
         # period.
         LogUpdateSize: 32MiB
 
+      ShellAccess:
+        # An admin user can use "arvados-client shell" to start an
+        # interactive shell (with any user ID) in any running
+        # container.
+        Admin: false
+
+        # Any user can use "arvados-client shell" to start an
+        # interactive shell (with any user ID) in any running
+        # container that they started, provided it isn't also
+        # associated with a different user's container request.
+        #
+        # Interactive sessions make it easy to alter the container's
+        # runtime environment in ways that aren't recorded or
+        # reproducible. Consider the implications for automatic
+        # container reuse before enabling and using this feature. In
+        # particular, note that starting an interactive session does
+        # not disqualify a container from being reused by a different
+        # user/workflow in the future.
+        User: false
+
       SLURM:
         PrioritySpread: 0
         SbatchArgumentsList: []
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 00523c7826..b86266d67e 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -336,6 +336,10 @@ func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOption
 	return conn.chooseBackend(options.UUID).ContainerUnlock(ctx, options)
 }
 
+func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ContainerSSHConnection, error) {
+	return conn.chooseBackend(options.UUID).ContainerSSH(ctx, options)
+}
+
 func (conn *Conn) ContainerRequestList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerRequestList, error) {
 	return conn.generated_ContainerRequestList(ctx, options)
 }
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index b04757ac33..5f6fb192e1 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -25,6 +25,7 @@ import (
 	"git.arvados.org/arvados.git/sdk/go/health"
 	"git.arvados.org/arvados.git/sdk/go/httpserver"
 	"github.com/jmoiron/sqlx"
+
 	// sqlx needs lib/pq to talk to PostgreSQL
 	_ "github.com/lib/pq"
 )
@@ -100,6 +101,7 @@ func (h *Handler) setup() {
 		mux.Handle("/arvados/v1/collections/", rtr)
 		mux.Handle("/arvados/v1/users", rtr)
 		mux.Handle("/arvados/v1/users/", rtr)
+		mux.Handle("/arvados/v1/connect/", rtr)
 		mux.Handle("/arvados/v1/container_requests", rtr)
 		mux.Handle("/arvados/v1/container_requests/", rtr)
 		mux.Handle("/login", rtr)
diff --git a/lib/controller/localdb/container_gateway.go b/lib/controller/localdb/container_gateway.go
new file mode 100644
index 0000000000..e509278773
--- /dev/null
+++ b/lib/controller/localdb/container_gateway.go
@@ -0,0 +1,174 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+	"bufio"
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/auth"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"git.arvados.org/arvados.git/sdk/go/httpserver"
+)
+
+// ContainerSSH returns a connection to the SSH server in the
+// appropriate crunch-run process on the worker node where the
+// specified container is running.
+//
+// If the returned error is nil, the caller is responsible for closing
+// sshconn.Conn.
+func (conn *Conn) ContainerSSH(ctx context.Context, opts arvados.ContainerSSHOptions) (sshconn arvados.ContainerSSHConnection, err error) {
+	user, err := conn.railsProxy.UserGetCurrent(ctx, arvados.GetOptions{})
+	if err != nil {
+		return
+	}
+	ctr, err := conn.railsProxy.ContainerGet(ctx, arvados.GetOptions{UUID: opts.UUID})
+	if err != nil {
+		return
+	}
+	ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{conn.cluster.SystemRootToken}})
+	if !user.IsAdmin || !conn.cluster.Containers.ShellAccess.Admin {
+		if !conn.cluster.Containers.ShellAccess.User {
+			err = httpserver.ErrorWithStatus(errors.New("shell access is disabled in config"), http.StatusServiceUnavailable)
+			return
+		}
+		var crs arvados.ContainerRequestList
+		crs, err = conn.railsProxy.ContainerRequestList(ctxRoot, arvados.ListOptions{Limit: -1, Filters: []arvados.Filter{{"container_uuid", "=", opts.UUID}}})
+		if err != nil {
+			return
+		}
+		for _, cr := range crs.Items {
+			if cr.ModifiedByUserUUID != user.UUID {
+				err = httpserver.ErrorWithStatus(errors.New("permission denied: container is associated with requests submitted by other users"), http.StatusForbidden)
+				return
+			}
+		}
+		if crs.ItemsAvailable != len(crs.Items) {
+			err = httpserver.ErrorWithStatus(errors.New("incomplete response while checking permission"), http.StatusInternalServerError)
+			return
+		}
+	}
+
+	switch ctr.State {
+	case arvados.ContainerStateQueued, arvados.ContainerStateLocked:
+		err = httpserver.ErrorWithStatus(fmt.Errorf("gateway is not available, container is %s", strings.ToLower(string(ctr.State))), http.StatusServiceUnavailable)
+		return
+	case arvados.ContainerStateRunning:
+		if ctr.GatewayAddress == "" {
+			err = httpserver.ErrorWithStatus(errors.New("container is running but gateway is not available"), http.StatusServiceUnavailable)
+			return
+		}
+	default:
+		err = httpserver.ErrorWithStatus(fmt.Errorf("gateway is not available, container is %s", strings.ToLower(string(ctr.State))), http.StatusGone)
+		return
+	}
+	// crunch-run uses a self-signed / unverifiable TLS
+	// certificate, so we use the following scheme to ensure we're
+	// not talking to a MITM.
+	//
+	// 1. Compute ctrKey = HMAC-SHA256(sysRootToken,ctrUUID) --
+	// this will be the same ctrKey that a-d-c supplied to
+	// crunch-run in the GatewayAuthSecret env var.
+	//
+	// 2. Compute requestAuth = HMAC-SHA256(ctrKey,serverCert) and
+	// send it to crunch-run as the X-Arvados-Authorization
+	// header, proving that we know ctrKey. (Note a MITM cannot
+	// replay the proof to a real crunch-run server, because the
+	// real crunch-run server would have a different cert.)
+	//
+	// 3. Compute respondAuth = HMAC-SHA256(ctrKey,requestAuth)
+	// and ensure the server returns it in the
+	// X-Arvados-Authorization-Response header, proving that the
+	// server knows ctrKey.
+	var requestAuth, respondAuth string
+	netconn, err := tls.Dial("tcp", ctr.GatewayAddress, &tls.Config{
+		InsecureSkipVerify: true,
+		VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+			if len(rawCerts) == 0 {
+				return errors.New("no certificate received, cannot compute authorization header")
+			}
+			h := hmac.New(sha256.New, []byte(conn.cluster.SystemRootToken))
+			fmt.Fprint(h, opts.UUID)
+			authKey := fmt.Sprintf("%x", h.Sum(nil))
+			h = hmac.New(sha256.New, []byte(authKey))
+			h.Write(rawCerts[0])
+			requestAuth = fmt.Sprintf("%x", h.Sum(nil))
+			h.Reset()
+			h.Write([]byte(requestAuth))
+			respondAuth = fmt.Sprintf("%x", h.Sum(nil))
+			return nil
+		},
+	})
+	if err != nil {
+		err = httpserver.ErrorWithStatus(err, http.StatusBadGateway)
+		return
+	}
+	if respondAuth == "" {
+		err = httpserver.ErrorWithStatus(errors.New("BUG: no respondAuth"), http.StatusInternalServerError)
+		return
+	}
+	bufr := bufio.NewReader(netconn)
+	bufw := bufio.NewWriter(netconn)
+
+	u := url.URL{
+		Scheme: "http",
+		Host:   ctr.GatewayAddress,
+		Path:   "/ssh",
+	}
+	bufw.WriteString("GET " + u.String() + " HTTP/1.1\r\n")
+	bufw.WriteString("Host: " + u.Host + "\r\n")
+	bufw.WriteString("Upgrade: ssh\r\n")
+	bufw.WriteString("X-Arvados-Target-Uuid: " + opts.UUID + "\r\n")
+	bufw.WriteString("X-Arvados-Authorization: " + requestAuth + "\r\n")
+	bufw.WriteString("X-Arvados-Detach-Keys: " + opts.DetachKeys + "\r\n")
+	bufw.WriteString("X-Arvados-Login-Username: " + opts.LoginUsername + "\r\n")
+	bufw.WriteString("\r\n")
+	bufw.Flush()
+	resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
+	if err != nil {
+		err = httpserver.ErrorWithStatus(fmt.Errorf("error reading http response from gateway: %w", err), http.StatusBadGateway)
+		netconn.Close()
+		return
+	}
+	if resp.Header.Get("X-Arvados-Authorization-Response") != respondAuth {
+		err = httpserver.ErrorWithStatus(errors.New("bad X-Arvados-Authorization-Response header"), http.StatusBadGateway)
+		netconn.Close()
+		return
+	}
+	if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
+		strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
+		err = httpserver.ErrorWithStatus(errors.New("bad upgrade"), http.StatusBadGateway)
+		netconn.Close()
+		return
+	}
+
+	if !ctr.InteractiveSessionStarted {
+		_, err = conn.railsProxy.ContainerUpdate(ctxRoot, arvados.UpdateOptions{
+			UUID: opts.UUID,
+			Attrs: map[string]interface{}{
+				"interactive_session_started": true,
+			},
+		})
+		if err != nil {
+			netconn.Close()
+			return
+		}
+	}
+
+	sshconn.Conn = netconn
+	sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
+	sshconn.Logger = ctxlog.FromContext(ctx)
+	return
+}
diff --git a/lib/controller/localdb/container_gateway_test.go b/lib/controller/localdb/container_gateway_test.go
new file mode 100644
index 0000000000..aff569b098
--- /dev/null
+++ b/lib/controller/localdb/container_gateway_test.go
@@ -0,0 +1,172 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"time"
+
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/lib/crunchrun"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"git.arvados.org/arvados.git/sdk/go/auth"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&ContainerGatewaySuite{})
+
+type ContainerGatewaySuite struct {
+	cluster *arvados.Cluster
+	localdb *Conn
+	ctx     context.Context
+	ctrUUID string
+	gw      *crunchrun.Gateway
+}
+
+func (s *ContainerGatewaySuite) TearDownSuite(c *check.C) {
+	// Undo any changes/additions to the user database so they
+	// don't affect subsequent tests.
+	arvadostest.ResetEnv()
+	c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
+	cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+	c.Assert(err, check.IsNil)
+	s.cluster, err = cfg.GetCluster("")
+	c.Assert(err, check.IsNil)
+	s.localdb = NewConn(s.cluster)
+	s.ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+	s.ctrUUID = arvadostest.QueuedContainerUUID
+
+	h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
+	fmt.Fprint(h, s.ctrUUID)
+	authKey := fmt.Sprintf("%x", h.Sum(nil))
+
+	s.gw = &crunchrun.Gateway{
+		DockerContainerID: new(string),
+		ContainerUUID:     s.ctrUUID,
+		AuthSecret:        authKey,
+		Address:           "localhost:0",
+		Log:               ctxlog.TestLogger(c),
+	}
+	c.Assert(s.gw.Start(), check.IsNil)
+	rootctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
+	_, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+		UUID: s.ctrUUID,
+		Attrs: map[string]interface{}{
+			"state": arvados.ContainerStateLocked}})
+	c.Assert(err, check.IsNil)
+	_, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+		UUID: s.ctrUUID,
+		Attrs: map[string]interface{}{
+			"state":           arvados.ContainerStateRunning,
+			"gateway_address": s.gw.Address}})
+	c.Assert(err, check.IsNil)
+}
+
+func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
+	s.cluster.Containers.ShellAccess.Admin = true
+	s.cluster.Containers.ShellAccess.User = true
+	_, err := arvadostest.DB(c, s.cluster).Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
+	c.Check(err, check.IsNil)
+}
+
+func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
+	for _, trial := range []struct {
+		configAdmin bool
+		configUser  bool
+		sendToken   string
+		errorCode   int
+	}{
+		{true, true, arvadostest.ActiveTokenV2, 0},
+		{true, false, arvadostest.ActiveTokenV2, 503},
+		{false, true, arvadostest.ActiveTokenV2, 0},
+		{false, false, arvadostest.ActiveTokenV2, 503},
+		{true, true, arvadostest.AdminToken, 0},
+		{true, false, arvadostest.AdminToken, 0},
+		{false, true, arvadostest.AdminToken, 403},
+		{false, false, arvadostest.AdminToken, 503},
+	} {
+		c.Logf("trial %#v", trial)
+		s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
+		s.cluster.Containers.ShellAccess.User = trial.configUser
+		ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{trial.sendToken}})
+		sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+		if trial.errorCode == 0 {
+			if !c.Check(err, check.IsNil) {
+				continue
+			}
+			if !c.Check(sshconn.Conn, check.NotNil) {
+				continue
+			}
+			sshconn.Conn.Close()
+		} else {
+			c.Check(err, check.NotNil)
+			err, ok := err.(interface{ HTTPStatus() int })
+			if c.Check(ok, check.Equals, true) {
+				c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
+			}
+		}
+	}
+}
+
+func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
+	c.Logf("connecting to %s", s.gw.Address)
+	sshconn, err := s.localdb.ContainerSSH(s.ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+	c.Assert(err, check.IsNil)
+	c.Assert(sshconn.Conn, check.NotNil)
+	defer sshconn.Conn.Close()
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+
+		// Receive text banner
+		buf := make([]byte, 12)
+		_, err := io.ReadFull(sshconn.Conn, buf)
+		c.Check(err, check.IsNil)
+		c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
+
+		// Send text banner
+		_, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
+		c.Check(err, check.IsNil)
+
+		// Receive binary
+		_, err = io.ReadFull(sshconn.Conn, buf[:4])
+		c.Check(err, check.IsNil)
+		c.Check(buf[:4], check.DeepEquals, []byte{0, 0, 1, 0xfc})
+
+		// If we can get this far into an SSH handshake...
+		c.Log("success, tunnel is working")
+	}()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		c.Fail()
+	}
+	ctr, err := s.localdb.ContainerGet(s.ctx, arvados.GetOptions{UUID: s.ctrUUID})
+	c.Check(err, check.IsNil)
+	c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
+}
+
+func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
+	c.Log("trying with no token")
+	ctx := auth.NewContext(context.Background(), &auth.Credentials{})
+	_, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+	c.Check(err, check.ErrorMatches, `.* 401 .*`)
+
+	c.Log("trying with anonymous token")
+	ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AnonymousToken}})
+	_, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+	c.Check(err, check.ErrorMatches, `.* 404 .*`)
+}
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 9fb2a0d32b..83c89d322a 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -221,6 +221,13 @@ func (rtr *router) addRoutes() {
 				return rtr.backend.ContainerUnlock(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
+		{
+			arvados.EndpointContainerSSH,
+			func() interface{} { return &arvados.ContainerSSHOptions{} },
+			func(ctx context.Context, opts interface{}) (interface{}, error) {
+				return rtr.backend.ContainerSSH(ctx, *opts.(*arvados.ContainerSSHOptions))
+			},
+		},
 		{
 			arvados.EndpointSpecimenCreate,
 			func() interface{} { return &arvados.CreateOptions{} },
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index d9d24260bb..3a19f4ab5a 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -5,6 +5,7 @@
 package rpc
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"crypto/tls"
@@ -12,6 +13,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
@@ -21,6 +23,7 @@ import (
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/auth"
+	"git.arvados.org/arvados.git/sdk/go/httpserver"
 )
 
 const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
@@ -302,6 +305,82 @@ func (conn *Conn) ContainerUnlock(ctx context.Context, options arvados.GetOption
 	return resp, err
 }
 
+// ContainerSSH returns a connection to the out-of-band SSH server for
+// a running container. If the returned error is nil, the caller is
+// responsible for closing sshconn.Conn.
+func (conn *Conn) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (sshconn arvados.ContainerSSHConnection, err error) {
+	addr := conn.baseURL.Host
+	if strings.Index(addr, ":") < 1 || (strings.Contains(addr, "::") && addr[0] != '[') {
+		// hostname or ::1 or 1::1
+		addr = net.JoinHostPort(addr, "https")
+	}
+	insecure := false
+	if tlsconf := conn.httpClient.Transport.(*http.Transport).TLSClientConfig; tlsconf != nil && tlsconf.InsecureSkipVerify {
+		insecure = true
+	}
+	netconn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: insecure})
+	if err != nil {
+		err = fmt.Errorf("tls.Dial: %w", err)
+		return
+	}
+	defer func() {
+		if err != nil {
+			netconn.Close()
+		}
+	}()
+	bufr := bufio.NewReader(netconn)
+	bufw := bufio.NewWriter(netconn)
+
+	u, err := conn.baseURL.Parse("/" + strings.Replace(arvados.EndpointContainerSSH.Path, "{uuid}", options.UUID, -1))
+	if err != nil {
+		err = fmt.Errorf("tls.Dial: %w", err)
+		return
+	}
+	u.RawQuery = url.Values{
+		"detach_keys":    {options.DetachKeys},
+		"login_username": {options.LoginUsername},
+	}.Encode()
+	tokens, err := conn.tokenProvider(ctx)
+	if err != nil {
+		return
+	} else if len(tokens) < 1 {
+		err = httpserver.ErrorWithStatus(errors.New("unauthorized"), http.StatusUnauthorized)
+		return
+	}
+	bufw.WriteString("GET " + u.String() + " HTTP/1.1\r\n")
+	bufw.WriteString("Authorization: Bearer " + tokens[0] + "\r\n")
+	bufw.WriteString("Host: " + u.Host + "\r\n")
+	bufw.WriteString("Upgrade: ssh\r\n")
+	bufw.WriteString("\r\n")
+	bufw.Flush()
+	resp, err := http.ReadResponse(bufr, &http.Request{Method: "GET"})
+	if err != nil {
+		err = fmt.Errorf("http.ReadResponse: %w", err)
+		return
+	}
+	if resp.StatusCode != http.StatusSwitchingProtocols {
+		defer resp.Body.Close()
+		body, _ := ioutil.ReadAll(resp.Body)
+		var message string
+		var errDoc httpserver.ErrorResponse
+		if err := json.Unmarshal(body, &errDoc); err == nil {
+			message = strings.Join(errDoc.Errors, "; ")
+		} else {
+			message = fmt.Sprintf("%q", body)
+		}
+		err = fmt.Errorf("server did not provide a tunnel: %s (HTTP %d)", message, resp.StatusCode)
+		return
+	}
+	if strings.ToLower(resp.Header.Get("Upgrade")) != "ssh" ||
+		strings.ToLower(resp.Header.Get("Connection")) != "upgrade" {
+		err = fmt.Errorf("bad response from server: Upgrade %q Connection %q", resp.Header.Get("Upgrade"), resp.Header.Get("Connection"))
+		return
+	}
+	sshconn.Conn = netconn
+	sshconn.Bufrw = &bufio.ReadWriter{Reader: bufr, Writer: bufw}
+	return
+}
+
 func (conn *Conn) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
 	ep := arvados.EndpointContainerRequestCreate
 	var resp arvados.ContainerRequest
diff --git a/lib/crunchrun/container_gateway.go b/lib/crunchrun/container_gateway.go
new file mode 100644
index 0000000000..1116c4bb12
--- /dev/null
+++ b/lib/crunchrun/container_gateway.go
@@ -0,0 +1,335 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"sync"
+	"syscall"
+
+	"git.arvados.org/arvados.git/lib/selfsigned"
+	"git.arvados.org/arvados.git/sdk/go/httpserver"
+	"github.com/creack/pty"
+	"github.com/google/shlex"
+	"golang.org/x/crypto/ssh"
+)
+
+type Gateway struct {
+	DockerContainerID *string
+	ContainerUUID     string
+	Address           string // listen host:port; if port=0, Start() will change it to the selected port
+	AuthSecret        string
+	Log               interface {
+		Printf(fmt string, args ...interface{})
+	}
+
+	sshConfig   ssh.ServerConfig
+	requestAuth string
+	respondAuth string
+}
+
+// startGatewayServer starts an http server that allows authenticated
+// clients to open an interactive "docker exec" session and (in
+// future) connect to tcp ports inside the docker container.
+func (gw *Gateway) Start() error {
+	gw.sshConfig = ssh.ServerConfig{
+		NoClientAuth: true,
+		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+			if c.User() == "_" {
+				return nil, nil
+			} else {
+				return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
+			}
+		},
+		PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
+			if c.User() == "_" {
+				return &ssh.Permissions{
+					Extensions: map[string]string{
+						"pubkey-fp": ssh.FingerprintSHA256(pubKey),
+					},
+				}, nil
+			} else {
+				return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
+			}
+		},
+	}
+	pvt, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return err
+	}
+	err = pvt.Validate()
+	if err != nil {
+		return err
+	}
+	signer, err := ssh.NewSignerFromKey(pvt)
+	if err != nil {
+		return err
+	}
+	gw.sshConfig.AddHostKey(signer)
+
+	// Address (typically provided by arvados-dispatch-cloud) is
+	// HOST:PORT where HOST is our IP address or hostname as seen
+	// from arvados-controller, and PORT is either the desired
+	// port where we should run our gateway server, or "0" if we
+	// should choose an available port.
+	host, port, err := net.SplitHostPort(gw.Address)
+	if err != nil {
+		return err
+	}
+	cert, err := selfsigned.CertGenerator{}.Generate()
+	if err != nil {
+		return err
+	}
+	h := hmac.New(sha256.New, []byte(gw.AuthSecret))
+	h.Write(cert.Certificate[0])
+	gw.requestAuth = fmt.Sprintf("%x", h.Sum(nil))
+	h.Reset()
+	h.Write([]byte(gw.requestAuth))
+	gw.respondAuth = fmt.Sprintf("%x", h.Sum(nil))
+
+	srv := &httpserver.Server{
+		Server: http.Server{
+			Handler: http.HandlerFunc(gw.handleSSH),
+			TLSConfig: &tls.Config{
+				Certificates: []tls.Certificate{cert},
+			},
+		},
+		Addr: ":" + port,
+	}
+	err = srv.Start()
+	if err != nil {
+		return err
+	}
+	// Get the port number we are listening on (the port might be
+	// "0" or a port name, in which case this will be different).
+	_, port, err = net.SplitHostPort(srv.Addr)
+	if err != nil {
+		return err
+	}
+	// When changing state to Running, we will set
+	// gateway_address to "HOST:PORT" where HOST is our
+	// external hostname/IP as provided by arvados-dispatch-cloud,
+	// and PORT is the port number we ended up listening on.
+	gw.Address = net.JoinHostPort(host, port)
+	return nil
+}
+
+// handleSSH connects to an SSH server that allows the caller to run
+// interactive commands as root (or any other desired user) inside the
+// container. The tunnel itself can only be created by an
+// authenticated caller, so the SSH server itself is wide open (any
+// password or key will be accepted).
+//
+// Requests must have path "/ssh" and the following headers:
+//
+// Connection: upgrade
+// Upgrade: ssh
+// X-Arvados-Target-Uuid: uuid of container
+// X-Arvados-Authorization: must match
+// hmac(AuthSecret,certfingerprint) (this prevents other containers
+// and shell nodes from connecting directly)
+//
+// Optional headers:
+//
+// X-Arvados-Detach-Keys: argument to "docker exec --detach-keys",
+// e.g., "ctrl-p,ctrl-q"
+// X-Arvados-Login-Username: argument to "docker exec --user": account
+// used to run command(s) inside the container.
+func (gw *Gateway) handleSSH(w http.ResponseWriter, req *http.Request) {
+	// In future we'll handle browser traffic too, but for now the
+	// only traffic we expect is an SSH tunnel from
+	// (*lib/controller/localdb.Conn)ContainerSSH()
+	if req.Method != "GET" || req.Header.Get("Upgrade") != "ssh" {
+		http.Error(w, "path not found", http.StatusNotFound)
+		return
+	}
+	if want := req.Header.Get("X-Arvados-Target-Uuid"); want != gw.ContainerUUID {
+		http.Error(w, fmt.Sprintf("misdirected request: meant for %q but received by crunch-run %q", want, gw.ContainerUUID), http.StatusBadGateway)
+		return
+	}
+	if req.Header.Get("X-Arvados-Authorization") != gw.requestAuth {
+		http.Error(w, "bad X-Arvados-Authorization header", http.StatusUnauthorized)
+		return
+	}
+	detachKeys := req.Header.Get("X-Arvados-Detach-Keys")
+	username := req.Header.Get("X-Arvados-Login-Username")
+	if username == "" {
+		username = "root"
+	}
+	hj, ok := w.(http.Hijacker)
+	if !ok {
+		http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
+		return
+	}
+	netconn, _, err := hj.Hijack()
+	if !ok {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	defer netconn.Close()
+	w.Header().Set("Connection", "upgrade")
+	w.Header().Set("Upgrade", "ssh")
+	w.Header().Set("X-Arvados-Authorization-Response", gw.respondAuth)
+	netconn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\n"))
+	w.Header().Write(netconn)
+	netconn.Write([]byte("\r\n"))
+
+	ctx := req.Context()
+
+	conn, newchans, reqs, err := ssh.NewServerConn(netconn, &gw.sshConfig)
+	if err != nil {
+		gw.Log.Printf("ssh.NewServerConn: %s", err)
+		return
+	}
+	defer conn.Close()
+	go ssh.DiscardRequests(reqs)
+	for newch := range newchans {
+		if newch.ChannelType() != "session" {
+			newch.Reject(ssh.UnknownChannelType, fmt.Sprintf("unsupported channel type %q", newch.ChannelType()))
+			continue
+		}
+		ch, reqs, err := newch.Accept()
+		if err != nil {
+			gw.Log.Printf("accept channel: %s", err)
+			return
+		}
+		var pty0, tty0 *os.File
+		go func() {
+			// Where to send errors/messages for the
+			// client to see
+			logw := io.Writer(ch.Stderr())
+			// How to end lines when sending
+			// errors/messages to the client (changes to
+			// \r\n when using a pty)
+			eol := "\n"
+			// Env vars to add to child process
+			termEnv := []string(nil)
+			for req := range reqs {
+				ok := false
+				switch req.Type {
+				case "shell", "exec":
+					ok = true
+					var payload struct {
+						Command string
+					}
+					ssh.Unmarshal(req.Payload, &payload)
+					execargs, err := shlex.Split(payload.Command)
+					if err != nil {
+						fmt.Fprintf(logw, "error parsing supplied command: %s"+eol, err)
+						return
+					}
+					if len(execargs) == 0 {
+						execargs = []string{"/bin/bash", "-login"}
+					}
+					go func() {
+						cmd := exec.CommandContext(ctx, "docker", "exec", "-i", "--detach-keys="+detachKeys, "--user="+username)
+						cmd.Stdin = ch
+						cmd.Stdout = ch
+						cmd.Stderr = ch.Stderr()
+						if tty0 != nil {
+							cmd.Args = append(cmd.Args, "-t")
+							cmd.Stdin = tty0
+							cmd.Stdout = tty0
+							cmd.Stderr = tty0
+							var wg sync.WaitGroup
+							defer wg.Wait()
+							wg.Add(2)
+							go func() { io.Copy(ch, pty0); wg.Done() }()
+							go func() { io.Copy(pty0, ch); wg.Done() }()
+							// Send our own debug messages to tty as well.
+							logw = tty0
+						}
+						cmd.Args = append(cmd.Args, *gw.DockerContainerID)
+						cmd.Args = append(cmd.Args, execargs...)
+						cmd.SysProcAttr = &syscall.SysProcAttr{
+							Setctty: tty0 != nil,
+							Setsid:  true,
+						}
+						cmd.Env = append(os.Environ(), termEnv...)
+						err := cmd.Run()
+						var resp struct {
+							Status uint32
+						}
+						if exiterr, ok := err.(*exec.ExitError); ok {
+							if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+								resp.Status = uint32(status.ExitStatus())
+							}
+						} else if err != nil {
+							// Propagate errors like `exec: "docker": executable file not found in $PATH`
+							fmt.Fprintln(ch.Stderr(), err)
+						}
+						errClose := ch.CloseWrite()
+						if resp.Status == 0 && (err != nil || errClose != nil) {
+							resp.Status = 1
+						}
+						ch.SendRequest("exit-status", false, ssh.Marshal(&resp))
+						ch.Close()
+					}()
+				case "pty-req":
+					eol = "\r\n"
+					p, t, err := pty.Open()
+					if err != nil {
+						fmt.Fprintf(ch.Stderr(), "pty failed: %s"+eol, err)
+						break
+					}
+					defer p.Close()
+					defer t.Close()
+					pty0, tty0 = p, t
+					ok = true
+					var payload struct {
+						Term string
+						Cols uint32
+						Rows uint32
+						X    uint32
+						Y    uint32
+					}
+					ssh.Unmarshal(req.Payload, &payload)
+					termEnv = []string{"TERM=" + payload.Term, "USE_TTY=1"}
+					err = pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
+					if err != nil {
+						fmt.Fprintf(logw, "pty-req: setsize failed: %s"+eol, err)
+					}
+				case "window-change":
+					var payload struct {
+						Cols uint32
+						Rows uint32
+						X    uint32
+						Y    uint32
+					}
+					ssh.Unmarshal(req.Payload, &payload)
+					err := pty.Setsize(pty0, &pty.Winsize{Rows: uint16(payload.Rows), Cols: uint16(payload.Cols), X: uint16(payload.X), Y: uint16(payload.Y)})
+					if err != nil {
+						fmt.Fprintf(logw, "window-change: setsize failed: %s"+eol, err)
+						break
+					}
+					ok = true
+				case "env":
+					// TODO: implement "env"
+					// requests by setting env
+					// vars in the docker-exec
+					// command (not docker-exec's
+					// own environment, which
+					// would be a gaping security
+					// hole).
+				default:
+					// fmt.Fprintf(logw, "declining %q req"+eol, req.Type)
+				}
+				if req.WantReply {
+					req.Reply(ok, nil)
+				}
+			}
+		}()
+	}
+}
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 6e7e6feba5..7d6fb4ed47 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -178,6 +178,8 @@ type ContainerRunner struct {
 	arvMountLog   *ThrottledLogger
 
 	containerWatchdogInterval time.Duration
+
+	gateway Gateway
 }
 
 // setupSignals sets up signal handling to gracefully terminate the underlying
@@ -1478,7 +1480,7 @@ func (runner *ContainerRunner) UpdateContainerRunning() error {
 		return ErrCancelled
 	}
 	return runner.DispatcherArvClient.Update("containers", runner.Container.UUID,
-		arvadosclient.Dict{"container": arvadosclient.Dict{"state": "Running"}}, nil)
+		arvadosclient.Dict{"container": arvadosclient.Dict{"state": "Running", "gateway_address": runner.gateway.Address}}, nil)
 }
 
 // ContainerToken returns the api_token the container (and any
@@ -1877,6 +1879,20 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
 		return 1
 	}
 
+	cr.gateway = Gateway{
+		Address:           os.Getenv("GatewayAddress"),
+		AuthSecret:        os.Getenv("GatewayAuthSecret"),
+		ContainerUUID:     containerID,
+		DockerContainerID: &cr.ContainerID,
+		Log:               cr.CrunchLog,
+	}
+	os.Unsetenv("GatewayAuthSecret")
+	err = cr.gateway.Start()
+	if err != nil {
+		log.Printf("error starting gateway server: %s", err)
+		return 1
+	}
+
 	parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerID+".")
 	if tmperr != nil {
 		log.Printf("%s: %v", containerID, tmperr)
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index a25ed60150..6a74280ca4 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -5,8 +5,10 @@
 package worker
 
 import (
+	"crypto/hmac"
 	"crypto/md5"
 	"crypto/rand"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
@@ -116,6 +118,7 @@ func NewPool(logger logrus.FieldLogger, arvClient *arvados.Client, reg *promethe
 		timeoutTERM:                    duration(cluster.Containers.CloudVMs.TimeoutTERM, defaultTimeoutTERM),
 		timeoutSignal:                  duration(cluster.Containers.CloudVMs.TimeoutSignal, defaultTimeoutSignal),
 		timeoutStaleRunLock:            duration(cluster.Containers.CloudVMs.TimeoutStaleRunLock, defaultTimeoutStaleRunLock),
+		systemRootToken:                cluster.SystemRootToken,
 		installPublicKey:               installPublicKey,
 		tagKeyPrefix:                   cluster.Containers.CloudVMs.TagKeyPrefix,
 		stop:                           make(chan bool),
@@ -154,6 +157,7 @@ type Pool struct {
 	timeoutTERM                    time.Duration
 	timeoutSignal                  time.Duration
 	timeoutStaleRunLock            time.Duration
+	systemRootToken                string
 	installPublicKey               ssh.PublicKey
 	tagKeyPrefix                   string
 
@@ -990,6 +994,12 @@ func (wp *Pool) waitUntilLoaded() {
 	}
 }
 
+func (wp *Pool) gatewayAuthSecret(uuid string) string {
+	h := hmac.New(sha256.New, []byte(wp.systemRootToken))
+	fmt.Fprint(h, uuid)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
 // Return a random string of n hexadecimal digits (n*4 random bits). n
 // must be even.
 func randomHex(n int) string {
diff --git a/lib/dispatchcloud/worker/runner.go b/lib/dispatchcloud/worker/runner.go
index 4752121342..0fd99aeeef 100644
--- a/lib/dispatchcloud/worker/runner.go
+++ b/lib/dispatchcloud/worker/runner.go
@@ -8,6 +8,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net"
 	"syscall"
 	"time"
 
@@ -48,6 +49,8 @@ func newRemoteRunner(uuid string, wkr *worker) *remoteRunner {
 		"ARVADOS_API_HOST":  wkr.wp.arvClient.APIHost,
 		"ARVADOS_API_TOKEN": wkr.wp.arvClient.AuthToken,
 		"InstanceType":      instJSON.String(),
+		"GatewayAddress":    net.JoinHostPort(wkr.instance.Address(), "0"),
+		"GatewayAuthSecret": wkr.wp.gatewayAuthSecret(uuid),
 	}
 	if wkr.wp.arvClient.Insecure {
 		env["ARVADOS_API_HOST_INSECURE"] = "1"
diff --git a/lib/selfsigned/cert.go b/lib/selfsigned/cert.go
new file mode 100644
index 0000000000..ae521dd17c
--- /dev/null
+++ b/lib/selfsigned/cert.go
@@ -0,0 +1,76 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package selfsigned
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"math/big"
+	"net"
+	"time"
+)
+
+type CertGenerator struct {
+	Bits  int
+	Hosts []string
+	IsCA  bool
+}
+
+func (gen CertGenerator) Generate() (cert tls.Certificate, err error) {
+	keyUsage := x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment
+	if gen.IsCA {
+		keyUsage |= x509.KeyUsageCertSign
+	}
+	notBefore := time.Now()
+	notAfter := time.Now().Add(time.Hour * 24 * 365)
+	snMax := new(big.Int).Lsh(big.NewInt(1), 128)
+	sn, err := rand.Int(rand.Reader, snMax)
+	if err != nil {
+		err = fmt.Errorf("Failed to generate serial number: %w", err)
+		return
+	}
+	template := x509.Certificate{
+		SerialNumber: sn,
+		Subject: pkix.Name{
+			Organization: []string{"N/A"},
+		},
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		KeyUsage:              keyUsage,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  gen.IsCA,
+	}
+	for _, h := range gen.Hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			template.IPAddresses = append(template.IPAddresses, ip)
+		} else {
+			template.DNSNames = append(template.DNSNames, h)
+		}
+	}
+	bits := gen.Bits
+	if bits == 0 {
+		bits = 4096
+	}
+	priv, err := rsa.GenerateKey(rand.Reader, bits)
+	if err != nil {
+		err = fmt.Errorf("error generating key: %w", err)
+		return
+	}
+	certder, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+	if err != nil {
+		err = fmt.Errorf("error creating certificate: %w", err)
+		return
+	}
+	cert = tls.Certificate{
+		Certificate: [][]byte{certder},
+		PrivateKey:  priv,
+	}
+	return
+}
diff --git a/lib/selfsigned/cert_test.go b/lib/selfsigned/cert_test.go
new file mode 100644
index 0000000000..16ed8bd913
--- /dev/null
+++ b/lib/selfsigned/cert_test.go
@@ -0,0 +1,26 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package selfsigned
+
+import (
+	"testing"
+)
+
+func TestCert(t *testing.T) {
+	cert, err := CertGenerator{Bits: 1024, Hosts: []string{"localhost"}, IsCA: false}.Generate()
+	if err != nil {
+		t.Error(err)
+	}
+	if len(cert.Certificate) < 1 {
+		t.Error("no certificate!")
+	}
+	cert, err = CertGenerator{Bits: 2048, Hosts: []string{"localhost"}, IsCA: true}.Generate()
+	if err != nil {
+		t.Error(err)
+	}
+	if len(cert.Certificate) < 1 {
+		t.Error("no certificate!")
+	}
+}
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index a11872971a..37a3e007b1 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -5,8 +5,12 @@
 package arvados
 
 import (
+	"bufio"
 	"context"
 	"encoding/json"
+	"net"
+
+	"github.com/sirupsen/logrus"
 )
 
 type APIEndpoint struct {
@@ -41,6 +45,7 @@ var (
 	EndpointContainerDelete               = APIEndpoint{"DELETE", "arvados/v1/containers/{uuid}", ""}
 	EndpointContainerLock                 = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/lock", ""}
 	EndpointContainerUnlock               = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/unlock", ""}
+	EndpointContainerSSH                  = APIEndpoint{"GET", "arvados/v1/connect/{uuid}/ssh", ""} // move to /containers after #17014 fixes routing
 	EndpointContainerRequestCreate        = APIEndpoint{"POST", "arvados/v1/container_requests", "container_request"}
 	EndpointContainerRequestUpdate        = APIEndpoint{"PATCH", "arvados/v1/container_requests/{uuid}", "container_request"}
 	EndpointContainerRequestGet           = APIEndpoint{"GET", "arvados/v1/container_requests/{uuid}", ""}
@@ -65,6 +70,18 @@ var (
 	EndpointAPIClientAuthorizationCurrent = APIEndpoint{"GET", "arvados/v1/api_client_authorizations/current", ""}
 )
 
+type ContainerSSHOptions struct {
+	UUID          string `json:"uuid"`
+	DetachKeys    string `json:"detach_keys"`
+	LoginUsername string `json:"login_username"`
+}
+
+type ContainerSSHConnection struct {
+	Conn   net.Conn           `json:"-"`
+	Bufrw  *bufio.ReadWriter  `json:"-"`
+	Logger logrus.FieldLogger `json:"-"`
+}
+
 type GetOptions struct {
 	UUID         string   `json:"uuid,omitempty"`
 	Select       []string `json:"select"`
@@ -180,6 +197,7 @@ type API interface {
 	ContainerDelete(ctx context.Context, options DeleteOptions) (Container, error)
 	ContainerLock(ctx context.Context, options GetOptions) (Container, error)
 	ContainerUnlock(ctx context.Context, options GetOptions) (Container, error)
+	ContainerSSH(ctx context.Context, options ContainerSSHOptions) (ContainerSSHConnection, error)
 	ContainerRequestCreate(ctx context.Context, options CreateOptions) (ContainerRequest, error)
 	ContainerRequestUpdate(ctx context.Context, options UpdateOptions) (ContainerRequest, error)
 	ContainerRequestGet(ctx context.Context, options GetOptions) (ContainerRequest, error)
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 52c75d5113..ea3cb6899e 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -9,6 +9,7 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -67,6 +68,10 @@ type Client struct {
 	dd *DiscoveryDocument
 
 	defaultRequestID string
+
+	// APIHost and AuthToken were loaded from ARVADOS_* env vars
+	// (used to customize "no host/token" error messages)
+	loadedFromEnv bool
 }
 
 // InsecureHTTPClient is the default http.Client used by a Client with
@@ -123,6 +128,7 @@ func NewClientFromEnv() *Client {
 		Insecure:        insecure,
 		KeepServiceURIs: svcs,
 		Timeout:         5 * time.Minute,
+		loadedFromEnv:   true,
 	}
 }
 
@@ -312,6 +318,13 @@ func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, m
 		// Ensure body is closed even if we error out early
 		defer body.Close()
 	}
+	if c.APIHost == "" {
+		if c.loadedFromEnv {
+			return errors.New("ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set")
+		} else {
+			return errors.New("arvados.Client cannot perform request: APIHost is not set")
+		}
+	}
 	urlString := c.apiURL(path)
 	urlValues, err := anythingToValues(params)
 	if err != nil {
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 9dc9e17dd8..8222035a33 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -429,6 +429,10 @@ type ContainersConfig struct {
 		LogUpdatePeriod              Duration
 		LogUpdateSize                ByteSize
 	}
+	ShellAccess struct {
+		Admin bool
+		User  bool
+	}
 	SLURM struct {
 		PrioritySpread             int64
 		SbatchArgumentsList        []string
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index 8d65bb4006..b8530db342 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -8,28 +8,30 @@ import "time"
 
 // Container is an arvados#container resource.
 type Container struct {
-	UUID                 string                 `json:"uuid"`
-	Etag                 string                 `json:"etag"`
-	CreatedAt            time.Time              `json:"created_at"`
-	ModifiedByClientUUID string                 `json:"modified_by_client_uuid"`
-	ModifiedByUserUUID   string                 `json:"modified_by_user_uuid"`
-	ModifiedAt           time.Time              `json:"modified_at"`
-	Command              []string               `json:"command"`
-	ContainerImage       string                 `json:"container_image"`
-	Cwd                  string                 `json:"cwd"`
-	Environment          map[string]string      `json:"environment"`
-	LockedByUUID         string                 `json:"locked_by_uuid"`
-	Mounts               map[string]Mount       `json:"mounts"`
-	Output               string                 `json:"output"`
-	OutputPath           string                 `json:"output_path"`
-	Priority             int64                  `json:"priority"`
-	RuntimeConstraints   RuntimeConstraints     `json:"runtime_constraints"`
-	State                ContainerState         `json:"state"`
-	SchedulingParameters SchedulingParameters   `json:"scheduling_parameters"`
-	ExitCode             int                    `json:"exit_code"`
-	RuntimeStatus        map[string]interface{} `json:"runtime_status"`
-	StartedAt            *time.Time             `json:"started_at"`  // nil if not yet started
-	FinishedAt           *time.Time             `json:"finished_at"` // nil if not yet finished
+	UUID                      string                 `json:"uuid"`
+	Etag                      string                 `json:"etag"`
+	CreatedAt                 time.Time              `json:"created_at"`
+	ModifiedByClientUUID      string                 `json:"modified_by_client_uuid"`
+	ModifiedByUserUUID        string                 `json:"modified_by_user_uuid"`
+	ModifiedAt                time.Time              `json:"modified_at"`
+	Command                   []string               `json:"command"`
+	ContainerImage            string                 `json:"container_image"`
+	Cwd                       string                 `json:"cwd"`
+	Environment               map[string]string      `json:"environment"`
+	LockedByUUID              string                 `json:"locked_by_uuid"`
+	Mounts                    map[string]Mount       `json:"mounts"`
+	Output                    string                 `json:"output"`
+	OutputPath                string                 `json:"output_path"`
+	Priority                  int64                  `json:"priority"`
+	RuntimeConstraints        RuntimeConstraints     `json:"runtime_constraints"`
+	State                     ContainerState         `json:"state"`
+	SchedulingParameters      SchedulingParameters   `json:"scheduling_parameters"`
+	ExitCode                  int                    `json:"exit_code"`
+	RuntimeStatus             map[string]interface{} `json:"runtime_status"`
+	StartedAt                 *time.Time             `json:"started_at"`  // nil if not yet started
+	FinishedAt                *time.Time             `json:"finished_at"` // nil if not yet finished
+	GatewayAddress            string                 `json:"gateway_address"`
+	InteractiveSessionStarted bool                   `json:"interactive_session_started"`
 }
 
 // ContainerRequest is an arvados#container_request resource.
diff --git a/sdk/go/arvados/container_gateway.go b/sdk/go/arvados/container_gateway.go
new file mode 100644
index 0000000000..00c98d572e
--- /dev/null
+++ b/sdk/go/arvados/container_gateway.go
@@ -0,0 +1,74 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"sync"
+
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/sirupsen/logrus"
+)
+
+func (sshconn ContainerSSHConnection) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	hj, ok := w.(http.Hijacker)
+	if !ok {
+		http.Error(w, "ResponseWriter does not support connection upgrade", http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Connection", "upgrade")
+	w.Header().Set("Upgrade", "ssh")
+	w.WriteHeader(http.StatusSwitchingProtocols)
+	conn, bufrw, err := hj.Hijack()
+	if err != nil {
+		ctxlog.FromContext(req.Context()).WithError(err).Error("error hijacking ResponseWriter")
+		return
+	}
+	defer conn.Close()
+
+	var bytesIn, bytesOut int64
+	var wg sync.WaitGroup
+	ctx, cancel := context.WithCancel(context.Background())
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer cancel()
+		n, err := io.CopyN(conn, sshconn.Bufrw, int64(sshconn.Bufrw.Reader.Buffered()))
+		bytesOut += n
+		if err == nil {
+			n, err = io.Copy(conn, sshconn.Conn)
+			bytesOut += n
+		}
+		if err != nil {
+			ctxlog.FromContext(req.Context()).WithError(err).Error("error copying downstream")
+		}
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer cancel()
+		n, err := io.CopyN(sshconn.Conn, bufrw, int64(bufrw.Reader.Buffered()))
+		bytesIn += n
+		if err == nil {
+			n, err = io.Copy(sshconn.Conn, conn)
+			bytesIn += n
+		}
+		if err != nil {
+			ctxlog.FromContext(req.Context()).WithError(err).Error("error copying upstream")
+		}
+	}()
+	<-ctx.Done()
+	if sshconn.Logger != nil {
+		go func() {
+			wg.Wait()
+			sshconn.Logger.WithFields(logrus.Fields{
+				"bytesIn":  bytesIn,
+				"bytesOut": bytesOut,
+			}).Info("closed connection")
+		}()
+	}
+}
diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go
index df3e46febd..930eabf27e 100644
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@ -105,6 +105,10 @@ func (as *APIStub) ContainerUnlock(ctx context.Context, options arvados.GetOptio
 	as.appendCall(ctx, as.ContainerUnlock, options)
 	return arvados.Container{}, as.Error
 }
+func (as *APIStub) ContainerSSH(ctx context.Context, options arvados.ContainerSSHOptions) (arvados.ContainerSSHConnection, error) {
+	as.appendCall(ctx, as.ContainerSSH, options)
+	return arvados.ContainerSSHConnection{}, as.Error
+}
 func (as *APIStub) ContainerRequestCreate(ctx context.Context, options arvados.CreateOptions) (arvados.ContainerRequest, error) {
 	as.appendCall(ctx, as.ContainerRequestCreate, options)
 	return arvados.ContainerRequest{}, as.Error
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 6a0f7d9f49..953021f0e7 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -802,6 +802,10 @@ def setup_config():
                         "GitInternalDir": os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'internal.git'),
                     },
                     "SupportedDockerImageFormats": {"v1": {}},
+                    "ShellAccess": {
+                        "Admin": True,
+                        "User": True,
+                    },
                 },
                 "Volumes": {
                     "zzzzz-nyw5e-%015d"%n: {
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index d01787cbc7..8feee77ff2 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -77,6 +77,8 @@ class Container < ArvadosModel
     t.add :runtime_user_uuid
     t.add :runtime_auth_scopes
     t.add :lock_count
+    t.add :gateway_address
+    t.add :interactive_session_started
   end
 
   # Supported states for a container
@@ -102,11 +104,11 @@ class Container < ArvadosModel
   end
 
   def self.full_text_searchable_columns
-    super - ["secret_mounts", "secret_mounts_md5", "runtime_token"]
+    super - ["secret_mounts", "secret_mounts_md5", "runtime_token", "gateway_address"]
   end
 
   def self.searchable_columns *args
-    super - ["secret_mounts_md5", "runtime_token"]
+    super - ["secret_mounts_md5", "runtime_token", "gateway_address"]
   end
 
   def logged_attributes
@@ -478,7 +480,10 @@ class Container < ArvadosModel
     when Running
       permitted.push :priority, *progress_attrs
       if self.state_changed?
-        permitted.push :started_at
+        permitted.push :started_at, :gateway_address
+      end
+      if !self.interactive_session_started_was
+        permitted.push :interactive_session_started
       end
 
     when Complete
diff --git a/services/api/db/migrate/20210108033940_add_gateway_address_to_containers.rb b/services/api/db/migrate/20210108033940_add_gateway_address_to_containers.rb
new file mode 100644
index 0000000000..8683b51901
--- /dev/null
+++ b/services/api/db/migrate/20210108033940_add_gateway_address_to_containers.rb
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class AddGatewayAddressToContainers < ActiveRecord::Migration[5.2]
+  def change
+    add_column :containers, :gateway_address, :string
+  end
+end
diff --git a/services/api/db/migrate/20210126183521_add_interactive_session_started_to_containers.rb b/services/api/db/migrate/20210126183521_add_interactive_session_started_to_containers.rb
new file mode 100644
index 0000000000..3fe23f64d8
--- /dev/null
+++ b/services/api/db/migrate/20210126183521_add_interactive_session_started_to_containers.rb
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class AddInteractiveSessionStartedToContainers < ActiveRecord::Migration[5.2]
+  def change
+    add_column :containers, :interactive_session_started, :boolean, null: false, default: false
+  end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 12a28c6c72..14eca609eb 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -521,7 +521,9 @@ CREATE TABLE public.containers (
     runtime_user_uuid text,
     runtime_auth_scopes jsonb,
     runtime_token text,
-    lock_count integer DEFAULT 0 NOT NULL
+    lock_count integer DEFAULT 0 NOT NULL,
+    gateway_address character varying,
+    interactive_session_started boolean DEFAULT false NOT NULL
 );
 
 
@@ -3187,6 +3189,8 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20200914203202'),
 ('20201103170213'),
 ('20201105190435'),
-('20201202174753');
+('20201202174753'),
+('20210108033940'),
+('20210126183521');
 
 
diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 773a9a8766..35e2b7ed1d 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -803,6 +803,8 @@ class ContainerTest < ActiveSupport::TestCase
     [Container::Running, {priority: 123456789}],
     [Container::Running, {runtime_status: {'error' => 'oops'}}],
     [Container::Running, {cwd: '/'}],
+    [Container::Running, {gateway_address: "172.16.0.1:12345"}],
+    [Container::Running, {interactive_session_started: true}],
     [Container::Complete, {state: Container::Cancelled}],
     [Container::Complete, {priority: 123456789}],
     [Container::Complete, {runtime_status: {'error' => 'oops'}}],