X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/14d8cfa18f28586ad296e3e598d0a2a536be0613..e7c3a477fc4f75321671a6f601cc07a9180e4646:/lib/controller/localdb/container_gateway_test.go
diff --git a/lib/controller/localdb/container_gateway_test.go b/lib/controller/localdb/container_gateway_test.go
index 2717604201..f7310e8de1 100644
--- a/lib/controller/localdb/container_gateway_test.go
+++ b/lib/controller/localdb/container_gateway_test.go
@@ -5,6 +5,7 @@
package localdb
import (
+ "bytes"
"context"
"crypto/hmac"
"crypto/sha256"
@@ -12,14 +13,26 @@ import (
"io"
"io/ioutil"
"net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
"time"
- "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/router"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/lib/crunchrun"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
"golang.org/x/crypto/ssh"
check "gopkg.in/check.v1"
)
@@ -27,43 +40,68 @@ import (
var _ = check.Suite(&ContainerGatewaySuite{})
type ContainerGatewaySuite struct {
- cluster *arvados.Cluster
- localdb *Conn
- ctx context.Context
+ localdbSuite
+ reqUUID string
ctrUUID string
+ srv *httptest.Server
gw *crunchrun.Gateway
}
-func (s *ContainerGatewaySuite) TearDownSuite(c *check.C) {
- // Undo any changes/additions to the user database so they
- // don't affect subsequent tests.
- arvadostest.ResetEnv()
- c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
-}
+func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
+ s.localdbSuite.SetUpTest(c)
-func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
+ cr, err := s.localdb.ContainerRequestCreate(s.userctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "command": []string{"echo", time.Now().Format(time.RFC3339Nano)},
+ "container_count_max": 1,
+ "container_image": "arvados/apitestfixture:latest",
+ "cwd": "/tmp",
+ "environment": map[string]string{},
+ "output_path": "/out",
+ "priority": 1,
+ "state": arvados.ContainerRequestStateCommitted,
+ "mounts": map[string]interface{}{
+ "/out": map[string]interface{}{
+ "kind": "tmp",
+ "capacity": 1000000,
+ },
+ },
+ "runtime_constraints": map[string]interface{}{
+ "vcpus": 1,
+ "ram": 2,
+ }}})
c.Assert(err, check.IsNil)
- s.localdb = NewConn(s.cluster)
- s.ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
-
- s.ctrUUID = arvadostest.QueuedContainerUUID
+ s.reqUUID = cr.UUID
+ s.ctrUUID = cr.ContainerUUID
h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
fmt.Fprint(h, s.ctrUUID)
authKey := fmt.Sprintf("%x", h.Sum(nil))
+ rtr := router.New(s.localdb, router.Config{})
+ s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
+ s.srv.StartTLS()
+ // the test setup doesn't use lib/service so
+ // service.URLFromContext() returns nothing -- instead, this
+ // is how we advertise our internal URL and enable
+ // proxy-to-other-controller mode,
+ forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
+ ac := &arvados.Client{
+ APIHost: s.srv.Listener.Addr().String(),
+ AuthToken: arvadostest.Dispatch1Token,
+ Insecure: true,
+ }
s.gw = &crunchrun.Gateway{
ContainerUUID: s.ctrUUID,
AuthSecret: authKey,
Address: "localhost:0",
Log: ctxlog.TestLogger(c),
Target: crunchrun.GatewayTargetStub{},
+ ArvadosClient: ac,
}
c.Assert(s.gw.Start(), check.IsNil)
- rootctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
+
+ rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
_, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
UUID: s.ctrUUID,
Attrs: map[string]interface{}{
@@ -75,15 +113,18 @@ func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
"state": arvados.ContainerStateRunning,
"gateway_address": s.gw.Address}})
c.Assert(err, check.IsNil)
-}
-func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
s.cluster.Containers.ShellAccess.Admin = true
s.cluster.Containers.ShellAccess.User = true
- _, err := arvadostest.DB(c, s.cluster).Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
+ _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
c.Check(err, check.IsNil)
}
+func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
+ s.srv.Close()
+ s.localdbSuite.TearDownTest(c)
+}
+
func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
for _, trial := range []struct {
configAdmin bool
@@ -103,7 +144,7 @@ func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
c.Logf("trial %#v", trial)
s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
s.cluster.Containers.ShellAccess.User = trial.configUser
- ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{trial.sendToken}})
+ ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
if trial.errorCode == 0 {
if !c.Check(err, check.IsNil) {
@@ -149,7 +190,7 @@ func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
}
c.Logf("connecting to %s", s.gw.Address)
- sshconn, err := s.localdb.ContainerSSH(s.ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+ sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
c.Assert(err, check.IsNil)
c.Assert(sshconn.Conn, check.NotNil)
defer sshconn.Conn.Close()
@@ -185,9 +226,302 @@ func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
}
}
+func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
+ files := map[string]string{
+ "stderr.txt": "hello world\n",
+ "a/b/c/d.html": "\n",
+ }
+ client := arvados.NewClientFromEnv()
+ ac, err := arvadosclient.New(client)
+ c.Assert(err, check.IsNil)
+ kc, err := keepclient.MakeKeepClient(ac)
+ c.Assert(err, check.IsNil)
+ cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
+ c.Assert(err, check.IsNil)
+ for name, content := range files {
+ for i, ch := range name {
+ if ch == '/' {
+ err := cfs.Mkdir("/"+name[:i], 0777)
+ c.Assert(err, check.IsNil)
+ }
+ }
+ f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
+ c.Assert(err, check.IsNil)
+ f.Write([]byte(content))
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ }
+ cfs.Sync()
+ s.gw.LogCollection = cfs
+}
+
+func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
+ rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
+ txt, err := s.gw.LogCollection.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "manifest_text": txt,
+ }})
+ c.Assert(err, check.IsNil)
+ _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ UUID: s.ctrUUID,
+ Attrs: map[string]interface{}{
+ "state": arvados.ContainerStateComplete,
+ "exit_code": 0,
+ "log": coll.PortableDataHash,
+ }})
+ c.Assert(err, check.IsNil)
+ updatedReq, err := s.localdb.ContainerRequestGet(rootctx, arvados.GetOptions{UUID: s.reqUUID})
+ c.Assert(err, check.IsNil)
+ c.Logf("container request log UUID is %s", updatedReq.LogUUID)
+ crLog, err := s.localdb.CollectionGet(rootctx, arvados.GetOptions{UUID: updatedReq.LogUUID, Select: []string{"manifest_text"}})
+ c.Assert(err, check.IsNil)
+ c.Logf("collection log manifest:\n%s", crLog.ManifestText)
+ // Ensure localdb can't circumvent the keep-web proxy test by
+ // getting content from the container gateway.
+ s.gw.LogCollection = nil
+}
+
+func (s *ContainerGatewaySuite) TestContainerRequestLogViaTunnel(c *check.C) {
+ forceProxyForTest = true
+ defer func() { forceProxyForTest = false }()
+
+ s.gw = s.setupGatewayWithTunnel(c)
+ s.setupLogCollection(c)
+
+ for _, broken := range []bool{false, true} {
+ c.Logf("broken=%v", broken)
+
+ if broken {
+ delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
+ } else {
+ s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
+ defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
+ }
+
+ r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID+"/stderr.txt", nil)
+ c.Assert(err, check.IsNil)
+ r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
+ handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
+ UUID: s.reqUUID,
+ WebDAVOptions: arvados.WebDAVOptions{
+ Method: "GET",
+ Header: r.Header,
+ Path: "/" + s.ctrUUID + "/stderr.txt",
+ },
+ })
+ if broken {
+ c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
+ continue
+ }
+ c.Check(err, check.IsNil)
+ c.Assert(handler, check.NotNil)
+ rec := httptest.NewRecorder()
+ handler.ServeHTTP(rec, r)
+ resp := rec.Result()
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Equals, "hello world\n")
+ }
+}
+
+func (s *ContainerGatewaySuite) TestContainerRequestLogViaGateway(c *check.C) {
+ s.setupLogCollection(c)
+ s.testContainerRequestLog(c)
+}
+
+func (s *ContainerGatewaySuite) TestContainerRequestLogViaKeepWeb(c *check.C) {
+ s.setupLogCollection(c)
+ s.saveLogAndCloseGateway(c)
+ s.testContainerRequestLog(c)
+}
+
+func (s *ContainerGatewaySuite) testContainerRequestLog(c *check.C) {
+ for _, trial := range []struct {
+ method string
+ path string
+ header http.Header
+ unauthenticated bool
+ expectStatus int
+ expectBodyRe string
+ expectHeader http.Header
+ }{
+ {
+ method: "GET",
+ path: s.ctrUUID + "/stderr.txt",
+ expectStatus: http.StatusOK,
+ expectBodyRe: "hello world\n",
+ expectHeader: http.Header{
+ "Content-Type": {"text/plain; charset=utf-8"},
+ },
+ },
+ {
+ method: "GET",
+ path: s.ctrUUID + "/stderr.txt",
+ header: http.Header{
+ "Range": {"bytes=-6"},
+ },
+ expectStatus: http.StatusPartialContent,
+ expectBodyRe: "world\n",
+ expectHeader: http.Header{
+ "Content-Type": {"text/plain; charset=utf-8"},
+ "Content-Range": {"bytes 6-11/12"},
+ },
+ },
+ {
+ method: "OPTIONS",
+ path: s.ctrUUID + "/stderr.txt",
+ expectStatus: http.StatusOK,
+ expectBodyRe: "",
+ expectHeader: http.Header{
+ "Dav": {"1, 2"},
+ "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
+ },
+ },
+ {
+ method: "OPTIONS",
+ path: s.ctrUUID + "/stderr.txt",
+ unauthenticated: true,
+ header: http.Header{
+ "Access-Control-Request-Method": {"POST"},
+ },
+ expectStatus: http.StatusOK,
+ expectBodyRe: "",
+ expectHeader: http.Header{
+ "Access-Control-Allow-Headers": {"Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control"},
+ "Access-Control-Allow-Methods": {"COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK"},
+ "Access-Control-Allow-Origin": {"*"},
+ "Access-Control-Max-Age": {"86400"},
+ },
+ },
+ {
+ method: "PROPFIND",
+ path: s.ctrUUID + "/",
+ expectStatus: http.StatusMultiStatus,
+ expectBodyRe: `.*\Qstderr.txt\E.*>\n?`,
+ expectHeader: http.Header{
+ "Content-Type": {"text/xml; charset=utf-8"},
+ },
+ },
+ {
+ method: "PROPFIND",
+ path: s.ctrUUID,
+ expectStatus: http.StatusMultiStatus,
+ expectBodyRe: `.*\Qstderr.txt\E.*>\n?`,
+ expectHeader: http.Header{
+ "Content-Type": {"text/xml; charset=utf-8"},
+ },
+ },
+ {
+ method: "PROPFIND",
+ path: s.ctrUUID + "/a/b/c/",
+ expectStatus: http.StatusMultiStatus,
+ expectBodyRe: `.*\Qd.html\E.*>\n?`,
+ expectHeader: http.Header{
+ "Content-Type": {"text/xml; charset=utf-8"},
+ },
+ },
+ {
+ method: "GET",
+ path: s.ctrUUID + "/a/b/c/d.html",
+ expectStatus: http.StatusOK,
+ expectBodyRe: "\n",
+ expectHeader: http.Header{
+ "Content-Type": {"text/html; charset=utf-8"},
+ },
+ },
+ } {
+ c.Logf("trial %#v", trial)
+ ctx := s.userctx
+ if trial.unauthenticated {
+ ctx = auth.NewContext(context.Background(), auth.CredentialsFromRequest(&http.Request{URL: &url.URL{}, Header: http.Header{}}))
+ }
+ r, err := http.NewRequestWithContext(ctx, trial.method, "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+trial.path, nil)
+ c.Assert(err, check.IsNil)
+ for k := range trial.header {
+ r.Header.Set(k, trial.header.Get(k))
+ }
+ handler, err := s.localdb.ContainerRequestLog(ctx, arvados.ContainerLogOptions{
+ UUID: s.reqUUID,
+ WebDAVOptions: arvados.WebDAVOptions{
+ Method: trial.method,
+ Header: r.Header,
+ Path: "/" + trial.path,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ c.Assert(handler, check.NotNil)
+ rec := httptest.NewRecorder()
+ handler.ServeHTTP(rec, r)
+ resp := rec.Result()
+ c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
+ for k := range trial.expectHeader {
+ c.Check(resp.Header.Get(k), check.Equals, trial.expectHeader.Get(k))
+ }
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Matches, trial.expectBodyRe)
+ }
+}
+
+func (s *ContainerGatewaySuite) TestContainerRequestLogViaCadaver(c *check.C) {
+ s.setupLogCollection(c)
+
+ out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "ls")
+ c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
+ c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
+
+ out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
+ c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
+
+ s.saveLogAndCloseGateway(c)
+
+ out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
+ c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
+}
+
+func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
+ // Replace s.srv with an HTTP server, otherwise cadaver will
+ // just fail on TLS cert verification.
+ s.srv.Close()
+ rtr := router.New(s.localdb, router.Config{})
+ s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
+ s.srv.Start()
+
+ tempdir, err := ioutil.TempDir("", "localdb-test-")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tempdir)
+
+ cmd := exec.Command("cadaver", s.srv.URL+path)
+ if password != "" {
+ cmd.Env = append(os.Environ(), "HOME="+tempdir)
+ f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+ c.Assert(err, check.IsNil)
+ _, err = fmt.Fprintf(f, "default login none password %s\n", password)
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
+ cmd.Stdin = bytes.NewBufferString(stdin)
+ cmd.Dir = tempdir
+ stdout, err := cmd.StdoutPipe()
+ c.Assert(err, check.Equals, nil)
+ cmd.Stderr = cmd.Stdout
+ c.Logf("cmd: %v", cmd.Args)
+ go cmd.Start()
+
+ var buf bytes.Buffer
+ _, err = io.Copy(&buf, stdout)
+ c.Check(err, check.Equals, nil)
+ err = cmd.Wait()
+ c.Check(err, check.Equals, nil)
+ return buf.String()
+}
+
func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
c.Logf("connecting to %s", s.gw.Address)
- sshconn, err := s.localdb.ContainerSSH(s.ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+ sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
c.Assert(err, check.IsNil)
c.Assert(sshconn.Conn, check.NotNil)
defer sshconn.Conn.Close()
@@ -218,19 +552,155 @@ func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
case <-time.After(time.Second):
c.Fail()
}
- ctr, err := s.localdb.ContainerGet(s.ctx, arvados.GetOptions{UUID: s.ctrUUID})
+ ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
c.Check(err, check.IsNil)
c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
}
func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
c.Log("trying with no token")
- ctx := auth.NewContext(context.Background(), &auth.Credentials{})
+ ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
_, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
c.Check(err, check.ErrorMatches, `.* 401 .*`)
c.Log("trying with anonymous token")
- ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AnonymousToken}})
+ ctx = ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
_, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
c.Check(err, check.ErrorMatches, `.* 404 .*`)
}
+
+func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
+ // no AuthSecret
+ conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
+ UUID: s.ctrUUID,
+ })
+ c.Check(err, check.ErrorMatches, `authentication error`)
+ c.Check(conn.Conn, check.IsNil)
+
+ // bogus AuthSecret
+ conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
+ UUID: s.ctrUUID,
+ AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ })
+ c.Check(err, check.ErrorMatches, `authentication error`)
+ c.Check(conn.Conn, check.IsNil)
+
+ // good AuthSecret
+ conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
+ UUID: s.ctrUUID,
+ AuthSecret: s.gw.AuthSecret,
+ })
+ c.Check(err, check.IsNil)
+ c.Check(conn.Conn, check.NotNil)
+}
+
+func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
+ forceProxyForTest = true
+ defer func() { forceProxyForTest = false }()
+ s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
+ defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
+ s.testConnectThroughTunnel(c, "")
+}
+
+func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
+ forceProxyForTest = true
+ defer func() { forceProxyForTest = false }()
+ // forceInternalURLForTest will not be usable because it isn't
+ // listed in s.cluster.Services.Controller.InternalURLs
+ s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
+}
+
+func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
+ s.testConnectThroughTunnel(c, "")
+}
+
+func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
+ rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
+ // Until the tunnel starts up, set gateway_address to a value
+ // that can't work. We want to ensure the only way we can
+ // reach the gateway is through the tunnel.
+ tungw := &crunchrun.Gateway{
+ ContainerUUID: s.ctrUUID,
+ AuthSecret: s.gw.AuthSecret,
+ Log: ctxlog.TestLogger(c),
+ Target: crunchrun.GatewayTargetStub{},
+ ArvadosClient: s.gw.ArvadosClient,
+ UpdateTunnelURL: func(url string) {
+ c.Logf("UpdateTunnelURL(%q)", url)
+ gwaddr := "tunnel " + url
+ s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ UUID: s.ctrUUID,
+ Attrs: map[string]interface{}{
+ "gateway_address": gwaddr}})
+ },
+ }
+ c.Assert(tungw.Start(), check.IsNil)
+
+ // We didn't supply an external hostname in the Address field,
+ // so Start() should assign a local address.
+ host, _, err := net.SplitHostPort(tungw.Address)
+ c.Assert(err, check.IsNil)
+ c.Check(host, check.Equals, "127.0.0.1")
+
+ _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ UUID: s.ctrUUID,
+ Attrs: map[string]interface{}{
+ "state": arvados.ContainerStateRunning,
+ }})
+ c.Assert(err, check.IsNil)
+
+ for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
+ ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
+ c.Assert(err, check.IsNil)
+ c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
+ c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
+ if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
+ break
+ }
+ }
+ return tungw
+}
+
+func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
+ s.setupGatewayWithTunnel(c)
+ c.Log("connecting to gateway through tunnel")
+ arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
+ sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
+ if expectErrorMatch != "" {
+ c.Check(err, check.ErrorMatches, expectErrorMatch)
+ return
+ }
+ c.Assert(err, check.IsNil)
+ c.Assert(sshconn.Conn, check.NotNil)
+ defer sshconn.Conn.Close()
+
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+
+ // Receive text banner
+ buf := make([]byte, 12)
+ _, err := io.ReadFull(sshconn.Conn, buf)
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
+
+ // Send text banner
+ _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
+ c.Check(err, check.IsNil)
+
+ // Receive binary
+ _, err = io.ReadFull(sshconn.Conn, buf[:4])
+ c.Check(err, check.IsNil)
+
+ // If we can get this far into an SSH handshake...
+ c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
+ }()
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ c.Fail()
+ }
+ ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
+ c.Check(err, check.IsNil)
+ c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
+}