lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bytes"
   9         "crypto/hmac"
  10         "crypto/sha256"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http"
  16         "net/http/httptest"
  17         "net/url"
  18         "os"
  19         "os/exec"
  20         "path/filepath"
  21         "strings"
  22         "time"
  23
  24         "git.arvados.org/arvados.git/lib/controller/router"
  25         "git.arvados.org/arvados.git/lib/controller/rpc"
  26         "git.arvados.org/arvados.git/lib/crunchrun"
  27         "git.arvados.org/arvados.git/lib/ctrlctx"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  30         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  31         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  32         "git.arvados.org/arvados.git/sdk/go/keepclient"
  33         "golang.org/x/crypto/ssh"
  34         check "gopkg.in/check.v1"
  35 )
  36
  37 var _ = check.Suite(&ContainerGatewaySuite{})
  38
  39 type ContainerGatewaySuite struct {
  40         localdbSuite
  41         ctrUUID string
  42         srv     *httptest.Server
  43         gw      *crunchrun.Gateway
  44 }
  45
  46 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
  47         s.localdbSuite.SetUpTest(c)
  48
  49         s.ctrUUID = arvadostest.QueuedContainerUUID
  50
  51         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
  52         fmt.Fprint(h, s.ctrUUID)
  53         authKey := fmt.Sprintf("%x", h.Sum(nil))
  54
  55         rtr := router.New(s.localdb, router.Config{})
  56         s.srv = httptest.NewUnstartedServer(rtr)
  57         s.srv.StartTLS()
  58         // the test setup doesn't use lib/service so
  59         // service.URLFromContext() returns nothing -- instead, this
  60         // is how we advertise our internal URL and enable
  61         // proxy-to-other-controller mode,
  62         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
  63         ac := &arvados.Client{
  64                 APIHost:   s.srv.Listener.Addr().String(),
  65                 AuthToken: arvadostest.Dispatch1Token,
  66                 Insecure:  true,
  67         }
  68         s.gw = &crunchrun.Gateway{
  69                 ContainerUUID: s.ctrUUID,
  70                 AuthSecret:    authKey,
  71                 Address:       "localhost:0",
  72                 Log:           ctxlog.TestLogger(c),
  73                 Target:        crunchrun.GatewayTargetStub{},
  74                 ArvadosClient: ac,
  75         }
  76         c.Assert(s.gw.Start(), check.IsNil)
  77         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
  78         // OK if this line fails (because state is already Running
  79         // from a previous test case) as long as the following line
  80         // succeeds:
  81         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  82                 UUID: s.ctrUUID,
  83                 Attrs: map[string]interface{}{
  84                         "state": arvados.ContainerStateLocked}})
  85         _, err := s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  86                 UUID: s.ctrUUID,
  87                 Attrs: map[string]interface{}{
  88                         "state":           arvados.ContainerStateRunning,
  89                         "gateway_address": s.gw.Address}})
  90         c.Assert(err, check.IsNil)
  91
  92         s.cluster.Containers.ShellAccess.Admin = true
  93         s.cluster.Containers.ShellAccess.User = true
  94         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
  95         c.Check(err, check.IsNil)
  96 }
  97
  98 func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
  99         s.srv.Close()
 100         s.localdbSuite.TearDownTest(c)
 101 }
 102
 103 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 104         for _, trial := range []struct {
 105                 configAdmin bool
 106                 configUser  bool
 107                 sendToken   string
 108                 errorCode   int
 109         }{
 110                 {true, true, arvadostest.ActiveTokenV2, 0},
 111                 {true, false, arvadostest.ActiveTokenV2, 503},
 112                 {false, true, arvadostest.ActiveTokenV2, 0},
 113                 {false, false, arvadostest.ActiveTokenV2, 503},
 114                 {true, true, arvadostest.AdminToken, 0},
 115                 {true, false, arvadostest.AdminToken, 0},
 116                 {false, true, arvadostest.AdminToken, 403},
 117                 {false, false, arvadostest.AdminToken, 503},
 118         } {
 119                 c.Logf("trial %#v", trial)
 120                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 121                 s.cluster.Containers.ShellAccess.User = trial.configUser
 122                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 123                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 124                 if trial.errorCode == 0 {
 125                         if !c.Check(err, check.IsNil) {
 126                                 continue
 127                         }
 128                         if !c.Check(sshconn.Conn, check.NotNil) {
 129                                 continue
 130                         }
 131                         sshconn.Conn.Close()
 132                 } else {
 133                         c.Check(err, check.NotNil)
 134                         err, ok := err.(interface{ HTTPStatus() int })
 135                         if c.Check(ok, check.Equals, true) {
 136                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 137                         }
 138                 }
 139         }
 140 }
 141
 142 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 143         // Set up servers on a few TCP ports
 144         var addrs []string
 145         for i := 0; i < 3; i++ {
 146                 ln, err := net.Listen("tcp", ":0")
 147                 c.Assert(err, check.IsNil)
 148                 defer ln.Close()
 149                 addrs = append(addrs, ln.Addr().String())
 150                 go func() {
 151                         for {
 152                                 conn, err := ln.Accept()
 153                                 if err != nil {
 154                                         return
 155                                 }
 156                                 var gotAddr string
 157                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 158                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 159                                 if gotAddr == ln.Addr().String() {
 160                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 161                                 }
 162                                 conn.Close()
 163                         }
 164                 }()
 165         }
 166
 167         c.Logf("connecting to %s", s.gw.Address)
 168         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 169         c.Assert(err, check.IsNil)
 170         c.Assert(sshconn.Conn, check.NotNil)
 171         defer sshconn.Conn.Close()
 172         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 173                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 174         })
 175         c.Assert(err, check.IsNil)
 176         client := ssh.NewClient(conn, chans, reqs)
 177         for _, expectAddr := range addrs {
 178                 _, port, err := net.SplitHostPort(expectAddr)
 179                 c.Assert(err, check.IsNil)
 180
 181                 c.Logf("trying foo:%s", port)
 182                 {
 183                         conn, err := client.Dial("tcp", "foo:"+port)
 184                         c.Assert(err, check.IsNil)
 185                         conn.SetDeadline(time.Now().Add(time.Second))
 186                         buf, err := ioutil.ReadAll(conn)
 187                         c.Check(err, check.IsNil)
 188                         c.Check(string(buf), check.Equals, "")
 189                 }
 190
 191                 c.Logf("trying localhost:%s", port)
 192                 {
 193                         conn, err := client.Dial("tcp", "localhost:"+port)
 194                         c.Assert(err, check.IsNil)
 195                         conn.SetDeadline(time.Now().Add(time.Second))
 196                         conn.Write([]byte(expectAddr + "\n"))
 197                         var gotAddr string
 198                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 199                         c.Check(gotAddr, check.Equals, expectAddr)
 200                 }
 201         }
 202 }
 203
 204 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C, files map[string]string) {
 205         client := arvados.NewClientFromEnv()
 206         ac, err := arvadosclient.New(client)
 207         c.Assert(err, check.IsNil)
 208         kc, err := keepclient.MakeKeepClient(ac)
 209         c.Assert(err, check.IsNil)
 210         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 211         c.Assert(err, check.IsNil)
 212         for name, content := range files {
 213                 for i, ch := range name {
 214                         if ch == '/' {
 215                                 err := cfs.Mkdir("/"+name[:i], 0777)
 216                                 c.Assert(err, check.IsNil)
 217                         }
 218                 }
 219                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 220                 c.Assert(err, check.IsNil)
 221                 f.Write([]byte(content))
 222                 err = f.Close()
 223                 c.Assert(err, check.IsNil)
 224         }
 225         cfs.Sync()
 226         s.gw.LogCollection = cfs
 227 }
 228
 229 func (s *ContainerGatewaySuite) TestContainerLogViaTunnel(c *check.C) {
 230         forceProxyForTest = true
 231         defer func() { forceProxyForTest = false }()
 232
 233         s.gw = s.setupGatewayWithTunnel(c)
 234         s.setupLogCollection(c, map[string]string{
 235                 "stderr.txt": "hello world\n",
 236         })
 237
 238         for _, broken := range []bool{false, true} {
 239                 c.Logf("broken=%v", broken)
 240
 241                 if broken {
 242                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 243                 } else {
 244                         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 245                         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 246                 }
 247
 248                 handler, err := s.localdb.ContainerLog(s.userctx, arvados.ContainerLogOptions{
 249                         UUID:          s.ctrUUID,
 250                         WebDAVOptions: arvados.WebDAVOptions{Path: "/stderr.txt"},
 251                 })
 252                 if broken {
 253                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 254                         continue
 255                 }
 256                 c.Check(err, check.IsNil)
 257                 c.Assert(handler, check.NotNil)
 258                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/containers/"+s.ctrUUID+"/log/stderr.txt", nil)
 259                 c.Assert(err, check.IsNil)
 260                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 261                 rec := httptest.NewRecorder()
 262                 handler.ServeHTTP(rec, r)
 263                 resp := rec.Result()
 264                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 265                 buf, err := ioutil.ReadAll(resp.Body)
 266                 c.Check(err, check.IsNil)
 267                 c.Check(string(buf), check.Equals, "hello world\n")
 268         }
 269 }
 270
 271 func (s *ContainerGatewaySuite) TestContainerLogViaGateway(c *check.C) {
 272         s.testContainerLog(c, true)
 273 }
 274
 275 func (s *ContainerGatewaySuite) TestContainerLogViaKeepWeb(c *check.C) {
 276         s.testContainerLog(c, false)
 277 }
 278
 279 func (s *ContainerGatewaySuite) testContainerLog(c *check.C, viaGateway bool) {
 280         s.setupLogCollection(c, map[string]string{
 281                 "stderr.txt":   "hello world\n",
 282                 "a/b/c/d.html": "<html></html>\n",
 283         })
 284         if !viaGateway {
 285                 rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 286                 txt, err := s.gw.LogCollection.MarshalManifest(".")
 287                 c.Assert(err, check.IsNil)
 288                 coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 289                         Attrs: map[string]interface{}{
 290                                 "manifest_text": txt,
 291                         }})
 292                 c.Assert(err, check.IsNil)
 293                 _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 294                         UUID: s.ctrUUID,
 295                         Attrs: map[string]interface{}{
 296                                 "log":             coll.PortableDataHash,
 297                                 "gateway_address": "",
 298                         }})
 299                 c.Assert(err, check.IsNil)
 300                 // gateway_address="" above already ensures localdb
 301                 // can't circumvent the keep-web proxy test by getting
 302                 // content from the container gateway; this is just
 303                 // extra insurance.
 304                 s.gw.LogCollection = nil
 305         }
 306         for _, trial := range []struct {
 307                 method       string
 308                 path         string
 309                 header       http.Header
 310                 expectStatus int
 311                 expectBodyRe string
 312                 expectHeader http.Header
 313         }{
 314                 {
 315                         method:       "GET",
 316                         path:         "/stderr.txt",
 317                         expectStatus: http.StatusOK,
 318                         expectBodyRe: "hello world\n",
 319                         expectHeader: http.Header{
 320                                 "Content-Type": {"text/plain; charset=utf-8"},
 321                         },
 322                 },
 323                 {
 324                         method: "GET",
 325                         path:   "/stderr.txt",
 326                         header: http.Header{
 327                                 "Range": {"bytes=-6"},
 328                         },
 329                         expectStatus: http.StatusPartialContent,
 330                         expectBodyRe: "world\n",
 331                         expectHeader: http.Header{
 332                                 "Content-Type":  {"text/plain; charset=utf-8"},
 333                                 "Content-Range": {"bytes 6-11/12"},
 334                         },
 335                 },
 336                 {
 337                         method:       "OPTIONS",
 338                         path:         "/stderr.txt",
 339                         expectStatus: http.StatusOK,
 340                         expectBodyRe: "",
 341                         expectHeader: http.Header{
 342                                 "Dav":   {"1, 2"},
 343                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 344                         },
 345                 },
 346                 {
 347                         method:       "PROPFIND",
 348                         path:         "",
 349                         expectStatus: http.StatusMultiStatus,
 350                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*`,
 351                         expectHeader: http.Header{
 352                                 "Content-Type": {"text/xml; charset=utf-8"},
 353                         },
 354                 },
 355                 {
 356                         method:       "PROPFIND",
 357                         path:         "/a/b/c/",
 358                         expectStatus: http.StatusMultiStatus,
 359                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*`,
 360                         expectHeader: http.Header{
 361                                 "Content-Type": {"text/xml; charset=utf-8"},
 362                         },
 363                 },
 364                 {
 365                         method:       "GET",
 366                         path:         "/a/b/c/d.html",
 367                         expectStatus: http.StatusOK,
 368                         expectBodyRe: "<html></html>\n",
 369                         expectHeader: http.Header{
 370                                 "Content-Type": {"text/html; charset=utf-8"},
 371                         },
 372                 },
 373         } {
 374                 c.Logf("trial %#v", trial)
 375                 handler, err := s.localdb.ContainerLog(s.userctx, arvados.ContainerLogOptions{
 376                         UUID:          s.ctrUUID,
 377                         WebDAVOptions: arvados.WebDAVOptions{Path: trial.path},
 378                 })
 379                 c.Assert(err, check.IsNil)
 380                 c.Assert(handler, check.NotNil)
 381                 r, err := http.NewRequestWithContext(s.userctx, trial.method, "https://controller.example/arvados/v1/containers/"+s.ctrUUID+"/log"+trial.path, nil)
 382                 c.Assert(err, check.IsNil)
 383                 for k := range trial.header {
 384                         r.Header.Set(k, trial.header.Get(k))
 385                 }
 386                 rec := httptest.NewRecorder()
 387                 handler.ServeHTTP(rec, r)
 388                 resp := rec.Result()
 389                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
 390                 for k := range trial.expectHeader {
 391                         c.Check(resp.Header.Get(k), check.Equals, trial.expectHeader.Get(k))
 392                 }
 393                 buf, err := ioutil.ReadAll(resp.Body)
 394                 c.Check(err, check.IsNil)
 395                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
 396         }
 397 }
 398
 399 func (s *ContainerGatewaySuite) TestContainerLogViaCadaver(c *check.C) {
 400         out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/containers/"+s.ctrUUID+"/log", "ls")
 401         c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
 402         c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
 403
 404         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/containers/"+s.ctrUUID+"/log", "get stderr.txt")
 405         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 406 }
 407
 408 func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
 409         // Replace s.srv with an HTTP server, otherwise cadaver will
 410         // just fail on TLS cert verification.
 411         s.srv.Close()
 412         rtr := router.New(s.localdb, router.Config{})
 413         s.srv = httptest.NewUnstartedServer(rtr)
 414         s.srv.Start()
 415
 416         s.setupLogCollection(c, map[string]string{
 417                 "stderr.txt":   "hello world\n",
 418                 "a/b/c/d.html": "<html></html>\n",
 419         })
 420
 421         tempdir, err := ioutil.TempDir("", "localdb-test-")
 422         c.Assert(err, check.IsNil)
 423         defer os.RemoveAll(tempdir)
 424
 425         cmd := exec.Command("cadaver", s.srv.URL+path)
 426         if password != "" {
 427                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 428                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 429                 c.Assert(err, check.IsNil)
 430                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 431                 c.Assert(err, check.IsNil)
 432                 c.Assert(f.Close(), check.IsNil)
 433         }
 434         cmd.Stdin = bytes.NewBufferString(stdin)
 435         cmd.Dir = tempdir
 436         stdout, err := cmd.StdoutPipe()
 437         c.Assert(err, check.Equals, nil)
 438         cmd.Stderr = cmd.Stdout
 439         c.Logf("cmd: %v", cmd.Args)
 440         go cmd.Start()
 441
 442         var buf bytes.Buffer
 443         _, err = io.Copy(&buf, stdout)
 444         c.Check(err, check.Equals, nil)
 445         err = cmd.Wait()
 446         c.Check(err, check.Equals, nil)
 447         return buf.String()
 448 }
 449
 450 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 451         c.Logf("connecting to %s", s.gw.Address)
 452         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 453         c.Assert(err, check.IsNil)
 454         c.Assert(sshconn.Conn, check.NotNil)
 455         defer sshconn.Conn.Close()
 456
 457         done := make(chan struct{})
 458         go func() {
 459                 defer close(done)
 460
 461                 // Receive text banner
 462                 buf := make([]byte, 12)
 463                 _, err := io.ReadFull(sshconn.Conn, buf)
 464                 c.Check(err, check.IsNil)
 465                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 466
 467                 // Send text banner
 468                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 469                 c.Check(err, check.IsNil)
 470
 471                 // Receive binary
 472                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 473                 c.Check(err, check.IsNil)
 474
 475                 // If we can get this far into an SSH handshake...
 476                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 477         }()
 478         select {
 479         case <-done:
 480         case <-time.After(time.Second):
 481                 c.Fail()
 482         }
 483         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 484         c.Check(err, check.IsNil)
 485         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 486 }
 487
 488 func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
 489         c.Log("trying with no token")
 490         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
 491         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 492         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 493
 494         c.Log("trying with anonymous token")
 495         ctx = ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
 496         _, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 497         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 498 }
 499
 500 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 501         // no AuthSecret
 502         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 503                 UUID: s.ctrUUID,
 504         })
 505         c.Check(err, check.ErrorMatches, `authentication error`)
 506         c.Check(conn.Conn, check.IsNil)
 507
 508         // bogus AuthSecret
 509         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 510                 UUID:       s.ctrUUID,
 511                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 512         })
 513         c.Check(err, check.ErrorMatches, `authentication error`)
 514         c.Check(conn.Conn, check.IsNil)
 515
 516         // good AuthSecret
 517         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 518                 UUID:       s.ctrUUID,
 519                 AuthSecret: s.gw.AuthSecret,
 520         })
 521         c.Check(err, check.IsNil)
 522         c.Check(conn.Conn, check.NotNil)
 523 }
 524
 525 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 526         forceProxyForTest = true
 527         defer func() { forceProxyForTest = false }()
 528         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 529         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 530         s.testConnectThroughTunnel(c, "")
 531 }
 532
 533 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 534         forceProxyForTest = true
 535         defer func() { forceProxyForTest = false }()
 536         // forceInternalURLForTest will not be usable because it isn't
 537         // listed in s.cluster.Services.Controller.InternalURLs
 538         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 539 }
 540
 541 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 542         s.testConnectThroughTunnel(c, "")
 543 }
 544
 545 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
 546         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 547         // Until the tunnel starts up, set gateway_address to a value
 548         // that can't work. We want to ensure the only way we can
 549         // reach the gateway is through the tunnel.
 550         tungw := &crunchrun.Gateway{
 551                 ContainerUUID: s.ctrUUID,
 552                 AuthSecret:    s.gw.AuthSecret,
 553                 Log:           ctxlog.TestLogger(c),
 554                 Target:        crunchrun.GatewayTargetStub{},
 555                 ArvadosClient: s.gw.ArvadosClient,
 556                 UpdateTunnelURL: func(url string) {
 557                         c.Logf("UpdateTunnelURL(%q)", url)
 558                         gwaddr := "tunnel " + url
 559                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 560                                 UUID: s.ctrUUID,
 561                                 Attrs: map[string]interface{}{
 562                                         "gateway_address": gwaddr}})
 563                 },
 564         }
 565         c.Assert(tungw.Start(), check.IsNil)
 566
 567         // We didn't supply an external hostname in the Address field,
 568         // so Start() should assign a local address.
 569         host, _, err := net.SplitHostPort(tungw.Address)
 570         c.Assert(err, check.IsNil)
 571         c.Check(host, check.Equals, "127.0.0.1")
 572
 573         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 574                 UUID: s.ctrUUID,
 575                 Attrs: map[string]interface{}{
 576                         "state": arvados.ContainerStateRunning,
 577                 }})
 578         c.Assert(err, check.IsNil)
 579
 580         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 581                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 582                 c.Assert(err, check.IsNil)
 583                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 584                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 585                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 586                         break
 587                 }
 588         }
 589         return tungw
 590 }
 591
 592 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 593         s.setupGatewayWithTunnel(c)
 594         c.Log("connecting to gateway through tunnel")
 595         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 596         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 597         if expectErrorMatch != "" {
 598                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 599                 return
 600         }
 601         c.Assert(err, check.IsNil)
 602         c.Assert(sshconn.Conn, check.NotNil)
 603         defer sshconn.Conn.Close()
 604
 605         done := make(chan struct{})
 606         go func() {
 607                 defer close(done)
 608
 609                 // Receive text banner
 610                 buf := make([]byte, 12)
 611                 _, err := io.ReadFull(sshconn.Conn, buf)
 612                 c.Check(err, check.IsNil)
 613                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 614
 615                 // Send text banner
 616                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 617                 c.Check(err, check.IsNil)
 618
 619                 // Receive binary
 620                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 621                 c.Check(err, check.IsNil)
 622
 623                 // If we can get this far into an SSH handshake...
 624                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 625         }()
 626         select {
 627         case <-done:
 628         case <-time.After(time.Second):
 629                 c.Fail()
 630         }
 631         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 632         c.Check(err, check.IsNil)
 633         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 634 }