lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bytes"
   9         "crypto/hmac"
  10         "crypto/sha256"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http"
  16         "net/http/httptest"
  17         "net/url"
  18         "os"
  19         "os/exec"
  20         "path/filepath"
  21         "strings"
  22         "time"
  23
  24         "git.arvados.org/arvados.git/lib/controller/router"
  25         "git.arvados.org/arvados.git/lib/controller/rpc"
  26         "git.arvados.org/arvados.git/lib/crunchrun"
  27         "git.arvados.org/arvados.git/lib/ctrlctx"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  30         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  31         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  32         "git.arvados.org/arvados.git/sdk/go/httpserver"
  33         "git.arvados.org/arvados.git/sdk/go/keepclient"
  34         "golang.org/x/crypto/ssh"
  35         check "gopkg.in/check.v1"
  36 )
  37
  38 var _ = check.Suite(&ContainerGatewaySuite{})
  39
  40 type ContainerGatewaySuite struct {
  41         localdbSuite
  42         ctrUUID string
  43         srv     *httptest.Server
  44         gw      *crunchrun.Gateway
  45 }
  46
  47 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
  48         s.localdbSuite.SetUpTest(c)
  49
  50         s.ctrUUID = arvadostest.QueuedContainerUUID
  51
  52         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
  53         fmt.Fprint(h, s.ctrUUID)
  54         authKey := fmt.Sprintf("%x", h.Sum(nil))
  55
  56         rtr := router.New(s.localdb, router.Config{})
  57         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
  58         s.srv.StartTLS()
  59         // the test setup doesn't use lib/service so
  60         // service.URLFromContext() returns nothing -- instead, this
  61         // is how we advertise our internal URL and enable
  62         // proxy-to-other-controller mode,
  63         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
  64         ac := &arvados.Client{
  65                 APIHost:   s.srv.Listener.Addr().String(),
  66                 AuthToken: arvadostest.Dispatch1Token,
  67                 Insecure:  true,
  68         }
  69         s.gw = &crunchrun.Gateway{
  70                 ContainerUUID: s.ctrUUID,
  71                 AuthSecret:    authKey,
  72                 Address:       "localhost:0",
  73                 Log:           ctxlog.TestLogger(c),
  74                 Target:        crunchrun.GatewayTargetStub{},
  75                 ArvadosClient: ac,
  76         }
  77         c.Assert(s.gw.Start(), check.IsNil)
  78         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
  79         // OK if this line fails (because state is already Running
  80         // from a previous test case) as long as the following line
  81         // succeeds:
  82         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  83                 UUID: s.ctrUUID,
  84                 Attrs: map[string]interface{}{
  85                         "state": arvados.ContainerStateLocked}})
  86         _, err := s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  87                 UUID: s.ctrUUID,
  88                 Attrs: map[string]interface{}{
  89                         "state":           arvados.ContainerStateRunning,
  90                         "gateway_address": s.gw.Address}})
  91         c.Assert(err, check.IsNil)
  92
  93         s.cluster.Containers.ShellAccess.Admin = true
  94         s.cluster.Containers.ShellAccess.User = true
  95         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
  96         c.Check(err, check.IsNil)
  97 }
  98
  99 func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
 100         s.srv.Close()
 101         s.localdbSuite.TearDownTest(c)
 102 }
 103
 104 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 105         for _, trial := range []struct {
 106                 configAdmin bool
 107                 configUser  bool
 108                 sendToken   string
 109                 errorCode   int
 110         }{
 111                 {true, true, arvadostest.ActiveTokenV2, 0},
 112                 {true, false, arvadostest.ActiveTokenV2, 503},
 113                 {false, true, arvadostest.ActiveTokenV2, 0},
 114                 {false, false, arvadostest.ActiveTokenV2, 503},
 115                 {true, true, arvadostest.AdminToken, 0},
 116                 {true, false, arvadostest.AdminToken, 0},
 117                 {false, true, arvadostest.AdminToken, 403},
 118                 {false, false, arvadostest.AdminToken, 503},
 119         } {
 120                 c.Logf("trial %#v", trial)
 121                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 122                 s.cluster.Containers.ShellAccess.User = trial.configUser
 123                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 124                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 125                 if trial.errorCode == 0 {
 126                         if !c.Check(err, check.IsNil) {
 127                                 continue
 128                         }
 129                         if !c.Check(sshconn.Conn, check.NotNil) {
 130                                 continue
 131                         }
 132                         sshconn.Conn.Close()
 133                 } else {
 134                         c.Check(err, check.NotNil)
 135                         err, ok := err.(interface{ HTTPStatus() int })
 136                         if c.Check(ok, check.Equals, true) {
 137                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 138                         }
 139                 }
 140         }
 141 }
 142
 143 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 144         // Set up servers on a few TCP ports
 145         var addrs []string
 146         for i := 0; i < 3; i++ {
 147                 ln, err := net.Listen("tcp", ":0")
 148                 c.Assert(err, check.IsNil)
 149                 defer ln.Close()
 150                 addrs = append(addrs, ln.Addr().String())
 151                 go func() {
 152                         for {
 153                                 conn, err := ln.Accept()
 154                                 if err != nil {
 155                                         return
 156                                 }
 157                                 var gotAddr string
 158                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 159                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 160                                 if gotAddr == ln.Addr().String() {
 161                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 162                                 }
 163                                 conn.Close()
 164                         }
 165                 }()
 166         }
 167
 168         c.Logf("connecting to %s", s.gw.Address)
 169         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 170         c.Assert(err, check.IsNil)
 171         c.Assert(sshconn.Conn, check.NotNil)
 172         defer sshconn.Conn.Close()
 173         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 174                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 175         })
 176         c.Assert(err, check.IsNil)
 177         client := ssh.NewClient(conn, chans, reqs)
 178         for _, expectAddr := range addrs {
 179                 _, port, err := net.SplitHostPort(expectAddr)
 180                 c.Assert(err, check.IsNil)
 181
 182                 c.Logf("trying foo:%s", port)
 183                 {
 184                         conn, err := client.Dial("tcp", "foo:"+port)
 185                         c.Assert(err, check.IsNil)
 186                         conn.SetDeadline(time.Now().Add(time.Second))
 187                         buf, err := ioutil.ReadAll(conn)
 188                         c.Check(err, check.IsNil)
 189                         c.Check(string(buf), check.Equals, "")
 190                 }
 191
 192                 c.Logf("trying localhost:%s", port)
 193                 {
 194                         conn, err := client.Dial("tcp", "localhost:"+port)
 195                         c.Assert(err, check.IsNil)
 196                         conn.SetDeadline(time.Now().Add(time.Second))
 197                         conn.Write([]byte(expectAddr + "\n"))
 198                         var gotAddr string
 199                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 200                         c.Check(gotAddr, check.Equals, expectAddr)
 201                 }
 202         }
 203 }
 204
 205 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
 206         files := map[string]string{
 207                 "stderr.txt":   "hello world\n",
 208                 "a/b/c/d.html": "<html></html>\n",
 209         }
 210         client := arvados.NewClientFromEnv()
 211         ac, err := arvadosclient.New(client)
 212         c.Assert(err, check.IsNil)
 213         kc, err := keepclient.MakeKeepClient(ac)
 214         c.Assert(err, check.IsNil)
 215         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 216         c.Assert(err, check.IsNil)
 217         for name, content := range files {
 218                 for i, ch := range name {
 219                         if ch == '/' {
 220                                 err := cfs.Mkdir("/"+name[:i], 0777)
 221                                 c.Assert(err, check.IsNil)
 222                         }
 223                 }
 224                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 225                 c.Assert(err, check.IsNil)
 226                 f.Write([]byte(content))
 227                 err = f.Close()
 228                 c.Assert(err, check.IsNil)
 229         }
 230         cfs.Sync()
 231         s.gw.LogCollection = cfs
 232 }
 233
 234 func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
 235         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 236         txt, err := s.gw.LogCollection.MarshalManifest(".")
 237         c.Assert(err, check.IsNil)
 238         coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 239                 Attrs: map[string]interface{}{
 240                         "manifest_text": txt,
 241                 }})
 242         c.Assert(err, check.IsNil)
 243         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 244                 UUID: s.ctrUUID,
 245                 Attrs: map[string]interface{}{
 246                         "log":             coll.PortableDataHash,
 247                         "gateway_address": "",
 248                 }})
 249         c.Assert(err, check.IsNil)
 250         // gateway_address="" above already ensures localdb
 251         // can't circumvent the keep-web proxy test by getting
 252         // content from the container gateway; this is just
 253         // extra insurance.
 254         s.gw.LogCollection = nil
 255 }
 256
 257 func (s *ContainerGatewaySuite) TestContainerLogViaTunnel(c *check.C) {
 258         forceProxyForTest = true
 259         defer func() { forceProxyForTest = false }()
 260
 261         s.gw = s.setupGatewayWithTunnel(c)
 262         s.setupLogCollection(c)
 263
 264         for _, broken := range []bool{false, true} {
 265                 c.Logf("broken=%v", broken)
 266
 267                 if broken {
 268                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 269                 } else {
 270                         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 271                         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 272                 }
 273
 274                 handler, err := s.localdb.ContainerLog(s.userctx, arvados.ContainerLogOptions{
 275                         UUID:          s.ctrUUID,
 276                         WebDAVOptions: arvados.WebDAVOptions{Path: "/stderr.txt"},
 277                 })
 278                 if broken {
 279                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 280                         continue
 281                 }
 282                 c.Check(err, check.IsNil)
 283                 c.Assert(handler, check.NotNil)
 284                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/containers/"+s.ctrUUID+"/log/stderr.txt", nil)
 285                 c.Assert(err, check.IsNil)
 286                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 287                 rec := httptest.NewRecorder()
 288                 handler.ServeHTTP(rec, r)
 289                 resp := rec.Result()
 290                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 291                 buf, err := ioutil.ReadAll(resp.Body)
 292                 c.Check(err, check.IsNil)
 293                 c.Check(string(buf), check.Equals, "hello world\n")
 294         }
 295 }
 296
 297 func (s *ContainerGatewaySuite) TestContainerLogViaGateway(c *check.C) {
 298         s.setupLogCollection(c)
 299         s.testContainerLog(c)
 300 }
 301
 302 func (s *ContainerGatewaySuite) TestContainerLogViaKeepWeb(c *check.C) {
 303         s.setupLogCollection(c)
 304         s.saveLogAndCloseGateway(c)
 305         s.testContainerLog(c)
 306 }
 307
 308 func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
 309         for _, trial := range []struct {
 310                 method       string
 311                 path         string
 312                 header       http.Header
 313                 expectStatus int
 314                 expectBodyRe string
 315                 expectHeader http.Header
 316         }{
 317                 {
 318                         method:       "GET",
 319                         path:         "/stderr.txt",
 320                         expectStatus: http.StatusOK,
 321                         expectBodyRe: "hello world\n",
 322                         expectHeader: http.Header{
 323                                 "Content-Type": {"text/plain; charset=utf-8"},
 324                         },
 325                 },
 326                 {
 327                         method: "GET",
 328                         path:   "/stderr.txt",
 329                         header: http.Header{
 330                                 "Range": {"bytes=-6"},
 331                         },
 332                         expectStatus: http.StatusPartialContent,
 333                         expectBodyRe: "world\n",
 334                         expectHeader: http.Header{
 335                                 "Content-Type":  {"text/plain; charset=utf-8"},
 336                                 "Content-Range": {"bytes 6-11/12"},
 337                         },
 338                 },
 339                 {
 340                         method:       "OPTIONS",
 341                         path:         "/stderr.txt",
 342                         expectStatus: http.StatusOK,
 343                         expectBodyRe: "",
 344                         expectHeader: http.Header{
 345                                 "Dav":   {"1, 2"},
 346                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 347                         },
 348                 },
 349                 {
 350                         method:       "PROPFIND",
 351                         path:         "",
 352                         expectStatus: http.StatusMultiStatus,
 353                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*`,
 354                         expectHeader: http.Header{
 355                                 "Content-Type": {"text/xml; charset=utf-8"},
 356                         },
 357                 },
 358                 {
 359                         method:       "PROPFIND",
 360                         path:         "/a/b/c/",
 361                         expectStatus: http.StatusMultiStatus,
 362                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*`,
 363                         expectHeader: http.Header{
 364                                 "Content-Type": {"text/xml; charset=utf-8"},
 365                         },
 366                 },
 367                 {
 368                         method:       "GET",
 369                         path:         "/a/b/c/d.html",
 370                         expectStatus: http.StatusOK,
 371                         expectBodyRe: "<html></html>\n",
 372                         expectHeader: http.Header{
 373                                 "Content-Type": {"text/html; charset=utf-8"},
 374                         },
 375                 },
 376         } {
 377                 c.Logf("trial %#v", trial)
 378                 handler, err := s.localdb.ContainerLog(s.userctx, arvados.ContainerLogOptions{
 379                         UUID:          s.ctrUUID,
 380                         WebDAVOptions: arvados.WebDAVOptions{Path: trial.path},
 381                 })
 382                 c.Assert(err, check.IsNil)
 383                 c.Assert(handler, check.NotNil)
 384                 r, err := http.NewRequestWithContext(s.userctx, trial.method, "https://controller.example/arvados/v1/containers/"+s.ctrUUID+"/log"+trial.path, nil)
 385                 c.Assert(err, check.IsNil)
 386                 for k := range trial.header {
 387                         r.Header.Set(k, trial.header.Get(k))
 388                 }
 389                 rec := httptest.NewRecorder()
 390                 handler.ServeHTTP(rec, r)
 391                 resp := rec.Result()
 392                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
 393                 for k := range trial.expectHeader {
 394                         c.Check(resp.Header.Get(k), check.Equals, trial.expectHeader.Get(k))
 395                 }
 396                 buf, err := ioutil.ReadAll(resp.Body)
 397                 c.Check(err, check.IsNil)
 398                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
 399         }
 400 }
 401
 402 func (s *ContainerGatewaySuite) TestContainerLogViaCadaver(c *check.C) {
 403         s.setupLogCollection(c)
 404
 405         out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/containers/"+s.ctrUUID+"/log", "ls")
 406         c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
 407         c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
 408
 409         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/containers/"+s.ctrUUID+"/log", "get stderr.txt")
 410         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 411
 412         s.saveLogAndCloseGateway(c)
 413
 414         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/containers/"+s.ctrUUID+"/log", "get stderr.txt")
 415         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 416 }
 417
 418 func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
 419         // Replace s.srv with an HTTP server, otherwise cadaver will
 420         // just fail on TLS cert verification.
 421         s.srv.Close()
 422         rtr := router.New(s.localdb, router.Config{})
 423         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
 424         s.srv.Start()
 425
 426         tempdir, err := ioutil.TempDir("", "localdb-test-")
 427         c.Assert(err, check.IsNil)
 428         defer os.RemoveAll(tempdir)
 429
 430         cmd := exec.Command("cadaver", s.srv.URL+path)
 431         if password != "" {
 432                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 433                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 434                 c.Assert(err, check.IsNil)
 435                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 436                 c.Assert(err, check.IsNil)
 437                 c.Assert(f.Close(), check.IsNil)
 438         }
 439         cmd.Stdin = bytes.NewBufferString(stdin)
 440         cmd.Dir = tempdir
 441         stdout, err := cmd.StdoutPipe()
 442         c.Assert(err, check.Equals, nil)
 443         cmd.Stderr = cmd.Stdout
 444         c.Logf("cmd: %v", cmd.Args)
 445         go cmd.Start()
 446
 447         var buf bytes.Buffer
 448         _, err = io.Copy(&buf, stdout)
 449         c.Check(err, check.Equals, nil)
 450         err = cmd.Wait()
 451         c.Check(err, check.Equals, nil)
 452         return buf.String()
 453 }
 454
 455 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 456         c.Logf("connecting to %s", s.gw.Address)
 457         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 458         c.Assert(err, check.IsNil)
 459         c.Assert(sshconn.Conn, check.NotNil)
 460         defer sshconn.Conn.Close()
 461
 462         done := make(chan struct{})
 463         go func() {
 464                 defer close(done)
 465
 466                 // Receive text banner
 467                 buf := make([]byte, 12)
 468                 _, err := io.ReadFull(sshconn.Conn, buf)
 469                 c.Check(err, check.IsNil)
 470                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 471
 472                 // Send text banner
 473                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 474                 c.Check(err, check.IsNil)
 475
 476                 // Receive binary
 477                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 478                 c.Check(err, check.IsNil)
 479
 480                 // If we can get this far into an SSH handshake...
 481                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 482         }()
 483         select {
 484         case <-done:
 485         case <-time.After(time.Second):
 486                 c.Fail()
 487         }
 488         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 489         c.Check(err, check.IsNil)
 490         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 491 }
 492
 493 func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
 494         c.Log("trying with no token")
 495         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
 496         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 497         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 498
 499         c.Log("trying with anonymous token")
 500         ctx = ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
 501         _, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 502         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 503 }
 504
 505 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 506         // no AuthSecret
 507         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 508                 UUID: s.ctrUUID,
 509         })
 510         c.Check(err, check.ErrorMatches, `authentication error`)
 511         c.Check(conn.Conn, check.IsNil)
 512
 513         // bogus AuthSecret
 514         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 515                 UUID:       s.ctrUUID,
 516                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 517         })
 518         c.Check(err, check.ErrorMatches, `authentication error`)
 519         c.Check(conn.Conn, check.IsNil)
 520
 521         // good AuthSecret
 522         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 523                 UUID:       s.ctrUUID,
 524                 AuthSecret: s.gw.AuthSecret,
 525         })
 526         c.Check(err, check.IsNil)
 527         c.Check(conn.Conn, check.NotNil)
 528 }
 529
 530 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 531         forceProxyForTest = true
 532         defer func() { forceProxyForTest = false }()
 533         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 534         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 535         s.testConnectThroughTunnel(c, "")
 536 }
 537
 538 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 539         forceProxyForTest = true
 540         defer func() { forceProxyForTest = false }()
 541         // forceInternalURLForTest will not be usable because it isn't
 542         // listed in s.cluster.Services.Controller.InternalURLs
 543         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 544 }
 545
 546 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 547         s.testConnectThroughTunnel(c, "")
 548 }
 549
 550 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
 551         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 552         // Until the tunnel starts up, set gateway_address to a value
 553         // that can't work. We want to ensure the only way we can
 554         // reach the gateway is through the tunnel.
 555         tungw := &crunchrun.Gateway{
 556                 ContainerUUID: s.ctrUUID,
 557                 AuthSecret:    s.gw.AuthSecret,
 558                 Log:           ctxlog.TestLogger(c),
 559                 Target:        crunchrun.GatewayTargetStub{},
 560                 ArvadosClient: s.gw.ArvadosClient,
 561                 UpdateTunnelURL: func(url string) {
 562                         c.Logf("UpdateTunnelURL(%q)", url)
 563                         gwaddr := "tunnel " + url
 564                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 565                                 UUID: s.ctrUUID,
 566                                 Attrs: map[string]interface{}{
 567                                         "gateway_address": gwaddr}})
 568                 },
 569         }
 570         c.Assert(tungw.Start(), check.IsNil)
 571
 572         // We didn't supply an external hostname in the Address field,
 573         // so Start() should assign a local address.
 574         host, _, err := net.SplitHostPort(tungw.Address)
 575         c.Assert(err, check.IsNil)
 576         c.Check(host, check.Equals, "127.0.0.1")
 577
 578         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 579                 UUID: s.ctrUUID,
 580                 Attrs: map[string]interface{}{
 581                         "state": arvados.ContainerStateRunning,
 582                 }})
 583         c.Assert(err, check.IsNil)
 584
 585         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 586                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 587                 c.Assert(err, check.IsNil)
 588                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 589                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 590                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 591                         break
 592                 }
 593         }
 594         return tungw
 595 }
 596
 597 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 598         s.setupGatewayWithTunnel(c)
 599         c.Log("connecting to gateway through tunnel")
 600         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 601         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 602         if expectErrorMatch != "" {
 603                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 604                 return
 605         }
 606         c.Assert(err, check.IsNil)
 607         c.Assert(sshconn.Conn, check.NotNil)
 608         defer sshconn.Conn.Close()
 609
 610         done := make(chan struct{})
 611         go func() {
 612                 defer close(done)
 613
 614                 // Receive text banner
 615                 buf := make([]byte, 12)
 616                 _, err := io.ReadFull(sshconn.Conn, buf)
 617                 c.Check(err, check.IsNil)
 618                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 619
 620                 // Send text banner
 621                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 622                 c.Check(err, check.IsNil)
 623
 624                 // Receive binary
 625                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 626                 c.Check(err, check.IsNil)
 627
 628                 // If we can get this far into an SSH handshake...
 629                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 630         }()
 631         select {
 632         case <-done:
 633         case <-time.After(time.Second):
 634                 c.Fail()
 635         }
 636         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 637         c.Check(err, check.IsNil)
 638         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 639 }