lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "crypto/hmac"
   9         "crypto/sha256"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net"
  14         "net/http"
  15         "net/http/httptest"
  16         "net/url"
  17         "os"
  18         "strings"
  19         "time"
  20
  21         "git.arvados.org/arvados.git/lib/controller/router"
  22         "git.arvados.org/arvados.git/lib/controller/rpc"
  23         "git.arvados.org/arvados.git/lib/crunchrun"
  24         "git.arvados.org/arvados.git/lib/ctrlctx"
  25         "git.arvados.org/arvados.git/sdk/go/arvados"
  26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  28         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  29         "git.arvados.org/arvados.git/sdk/go/keepclient"
  30         "golang.org/x/crypto/ssh"
  31         check "gopkg.in/check.v1"
  32 )
  33
  34 var _ = check.Suite(&ContainerGatewaySuite{})
  35
  36 type ContainerGatewaySuite struct {
  37         localdbSuite
  38         ctrUUID string
  39         gw      *crunchrun.Gateway
  40 }
  41
  42 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
  43         s.localdbSuite.SetUpTest(c)
  44
  45         s.ctrUUID = arvadostest.QueuedContainerUUID
  46
  47         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
  48         fmt.Fprint(h, s.ctrUUID)
  49         authKey := fmt.Sprintf("%x", h.Sum(nil))
  50
  51         rtr := router.New(s.localdb, router.Config{})
  52         srv := httptest.NewUnstartedServer(rtr)
  53         srv.StartTLS()
  54         // the test setup doesn't use lib/service so
  55         // service.URLFromContext() returns nothing -- instead, this
  56         // is how we advertise our internal URL and enable
  57         // proxy-to-other-controller mode,
  58         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: srv.Listener.Addr().String()}
  59         ac := &arvados.Client{
  60                 APIHost:   srv.Listener.Addr().String(),
  61                 AuthToken: arvadostest.Dispatch1Token,
  62                 Insecure:  true,
  63         }
  64         s.gw = &crunchrun.Gateway{
  65                 ContainerUUID: s.ctrUUID,
  66                 AuthSecret:    authKey,
  67                 Address:       "localhost:0",
  68                 Log:           ctxlog.TestLogger(c),
  69                 Target:        crunchrun.GatewayTargetStub{},
  70                 ArvadosClient: ac,
  71         }
  72         c.Assert(s.gw.Start(), check.IsNil)
  73         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
  74         // OK if this line fails (because state is already Running
  75         // from a previous test case) as long as the following line
  76         // succeeds:
  77         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  78                 UUID: s.ctrUUID,
  79                 Attrs: map[string]interface{}{
  80                         "state": arvados.ContainerStateLocked}})
  81         _, err := s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  82                 UUID: s.ctrUUID,
  83                 Attrs: map[string]interface{}{
  84                         "state":           arvados.ContainerStateRunning,
  85                         "gateway_address": s.gw.Address}})
  86         c.Assert(err, check.IsNil)
  87
  88         s.cluster.Containers.ShellAccess.Admin = true
  89         s.cluster.Containers.ShellAccess.User = true
  90         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
  91         c.Check(err, check.IsNil)
  92 }
  93
  94 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
  95         for _, trial := range []struct {
  96                 configAdmin bool
  97                 configUser  bool
  98                 sendToken   string
  99                 errorCode   int
 100         }{
 101                 {true, true, arvadostest.ActiveTokenV2, 0},
 102                 {true, false, arvadostest.ActiveTokenV2, 503},
 103                 {false, true, arvadostest.ActiveTokenV2, 0},
 104                 {false, false, arvadostest.ActiveTokenV2, 503},
 105                 {true, true, arvadostest.AdminToken, 0},
 106                 {true, false, arvadostest.AdminToken, 0},
 107                 {false, true, arvadostest.AdminToken, 403},
 108                 {false, false, arvadostest.AdminToken, 503},
 109         } {
 110                 c.Logf("trial %#v", trial)
 111                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 112                 s.cluster.Containers.ShellAccess.User = trial.configUser
 113                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 114                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 115                 if trial.errorCode == 0 {
 116                         if !c.Check(err, check.IsNil) {
 117                                 continue
 118                         }
 119                         if !c.Check(sshconn.Conn, check.NotNil) {
 120                                 continue
 121                         }
 122                         sshconn.Conn.Close()
 123                 } else {
 124                         c.Check(err, check.NotNil)
 125                         err, ok := err.(interface{ HTTPStatus() int })
 126                         if c.Check(ok, check.Equals, true) {
 127                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 128                         }
 129                 }
 130         }
 131 }
 132
 133 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 134         // Set up servers on a few TCP ports
 135         var addrs []string
 136         for i := 0; i < 3; i++ {
 137                 ln, err := net.Listen("tcp", ":0")
 138                 c.Assert(err, check.IsNil)
 139                 defer ln.Close()
 140                 addrs = append(addrs, ln.Addr().String())
 141                 go func() {
 142                         for {
 143                                 conn, err := ln.Accept()
 144                                 if err != nil {
 145                                         return
 146                                 }
 147                                 var gotAddr string
 148                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 149                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 150                                 if gotAddr == ln.Addr().String() {
 151                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 152                                 }
 153                                 conn.Close()
 154                         }
 155                 }()
 156         }
 157
 158         c.Logf("connecting to %s", s.gw.Address)
 159         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 160         c.Assert(err, check.IsNil)
 161         c.Assert(sshconn.Conn, check.NotNil)
 162         defer sshconn.Conn.Close()
 163         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 164                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 165         })
 166         c.Assert(err, check.IsNil)
 167         client := ssh.NewClient(conn, chans, reqs)
 168         for _, expectAddr := range addrs {
 169                 _, port, err := net.SplitHostPort(expectAddr)
 170                 c.Assert(err, check.IsNil)
 171
 172                 c.Logf("trying foo:%s", port)
 173                 {
 174                         conn, err := client.Dial("tcp", "foo:"+port)
 175                         c.Assert(err, check.IsNil)
 176                         conn.SetDeadline(time.Now().Add(time.Second))
 177                         buf, err := ioutil.ReadAll(conn)
 178                         c.Check(err, check.IsNil)
 179                         c.Check(string(buf), check.Equals, "")
 180                 }
 181
 182                 c.Logf("trying localhost:%s", port)
 183                 {
 184                         conn, err := client.Dial("tcp", "localhost:"+port)
 185                         c.Assert(err, check.IsNil)
 186                         conn.SetDeadline(time.Now().Add(time.Second))
 187                         conn.Write([]byte(expectAddr + "\n"))
 188                         var gotAddr string
 189                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 190                         c.Check(gotAddr, check.Equals, expectAddr)
 191                 }
 192         }
 193 }
 194
 195 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C, files map[string]string) {
 196         client := arvados.NewClientFromEnv()
 197         ac, err := arvadosclient.New(client)
 198         c.Assert(err, check.IsNil)
 199         kc, err := keepclient.MakeKeepClient(ac)
 200         c.Assert(err, check.IsNil)
 201         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 202         c.Assert(err, check.IsNil)
 203         for name, content := range files {
 204                 for i, ch := range name {
 205                         if ch == '/' {
 206                                 err := cfs.Mkdir("/"+name[:i], 0777)
 207                                 c.Assert(err, check.IsNil)
 208                         }
 209                 }
 210                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 211                 c.Assert(err, check.IsNil)
 212                 f.Write([]byte(content))
 213                 err = f.Close()
 214                 c.Assert(err, check.IsNil)
 215         }
 216         cfs.Sync()
 217         s.gw.LogCollection = cfs
 218 }
 219
 220 func (s *ContainerGatewaySuite) TestContainerLogViaTunnel(c *check.C) {
 221         forceProxyForTest = true
 222         defer func() { forceProxyForTest = false }()
 223
 224         s.gw = s.setupGatewayWithTunnel(c)
 225         s.setupLogCollection(c, map[string]string{
 226                 "stderr.txt": "hello world\n",
 227         })
 228
 229         for _, broken := range []bool{false, true} {
 230                 c.Logf("broken=%v", broken)
 231
 232                 if broken {
 233                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 234                 } else {
 235                         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 236                         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 237                 }
 238
 239                 handler, err := s.localdb.ContainerLog(s.userctx, arvados.ContainerLogOptions{
 240                         UUID:          s.ctrUUID,
 241                         WebDAVOptions: arvados.WebDAVOptions{Path: "/stderr.txt"},
 242                 })
 243                 if broken {
 244                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 245                         continue
 246                 }
 247                 c.Check(err, check.IsNil)
 248                 c.Assert(handler, check.NotNil)
 249                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/containers/"+s.ctrUUID+"/log/stderr.txt", nil)
 250                 c.Assert(err, check.IsNil)
 251                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 252                 rec := httptest.NewRecorder()
 253                 handler.ServeHTTP(rec, r)
 254                 resp := rec.Result()
 255                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 256                 buf, err := ioutil.ReadAll(resp.Body)
 257                 c.Check(err, check.IsNil)
 258                 c.Check(string(buf), check.Equals, "hello world\n")
 259         }
 260 }
 261
 262 func (s *ContainerGatewaySuite) TestContainerLogViaGateway(c *check.C) {
 263         s.testContainerLog(c, true)
 264 }
 265
 266 func (s *ContainerGatewaySuite) TestContainerLogViaKeepWeb(c *check.C) {
 267         s.testContainerLog(c, false)
 268 }
 269
 270 func (s *ContainerGatewaySuite) testContainerLog(c *check.C, viaGateway bool) {
 271         s.setupLogCollection(c, map[string]string{
 272                 "stderr.txt":   "hello world\n",
 273                 "a/b/c/d.html": "<html></html>\n",
 274         })
 275         if !viaGateway {
 276                 rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 277                 txt, err := s.gw.LogCollection.MarshalManifest(".")
 278                 c.Assert(err, check.IsNil)
 279                 coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 280                         Attrs: map[string]interface{}{
 281                                 "manifest_text": txt,
 282                         }})
 283                 c.Assert(err, check.IsNil)
 284                 _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 285                         UUID: s.ctrUUID,
 286                         Attrs: map[string]interface{}{
 287                                 "log":             coll.PortableDataHash,
 288                                 "gateway_address": "",
 289                         }})
 290                 c.Assert(err, check.IsNil)
 291                 // _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 292                 //      UUID: s.ctrUUID,
 293                 //      Attrs: map[string]interface{}{
 294                 //              "state": "Cancelled",
 295                 //      }})
 296                 // c.Assert(err, check.IsNil)
 297                 // gateway_address="" above already ensures localdb
 298                 // can't circumvent the keep-web proxy test by getting
 299                 // content from the container gateway; this is just
 300                 // extra insurance.
 301                 s.gw.LogCollection = nil
 302         }
 303         for _, trial := range []struct {
 304                 method       string
 305                 path         string
 306                 header       http.Header
 307                 expectStatus int
 308                 expectBodyRe string
 309                 expectHeader http.Header
 310         }{
 311                 {
 312                         method:       "GET",
 313                         path:         "/stderr.txt",
 314                         expectStatus: http.StatusOK,
 315                         expectBodyRe: "hello world\n",
 316                         expectHeader: http.Header{
 317                                 "Content-Type": {"text/plain; charset=utf-8"},
 318                         },
 319                 },
 320                 {
 321                         method: "GET",
 322                         path:   "/stderr.txt",
 323                         header: http.Header{
 324                                 "Range": {"bytes=-6"},
 325                         },
 326                         expectStatus: http.StatusPartialContent,
 327                         expectBodyRe: "world\n",
 328                         expectHeader: http.Header{
 329                                 "Content-Type":  {"text/plain; charset=utf-8"},
 330                                 "Content-Range": {"bytes 6-11/12"},
 331                         },
 332                 },
 333                 {
 334                         method:       "OPTIONS",
 335                         path:         "/stderr.txt",
 336                         expectStatus: http.StatusOK,
 337                         expectBodyRe: "",
 338                         expectHeader: http.Header{
 339                                 "Dav":   {"1, 2"},
 340                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 341                         },
 342                 },
 343                 {
 344                         method:       "PROPFIND",
 345                         path:         "",
 346                         expectStatus: http.StatusMultiStatus,
 347                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*`,
 348                         expectHeader: http.Header{
 349                                 "Content-Type": {"text/xml; charset=utf-8"},
 350                         },
 351                 },
 352                 {
 353                         method:       "PROPFIND",
 354                         path:         "/a/b/c/",
 355                         expectStatus: http.StatusMultiStatus,
 356                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*`,
 357                         expectHeader: http.Header{
 358                                 "Content-Type": {"text/xml; charset=utf-8"},
 359                         },
 360                 },
 361                 {
 362                         method:       "GET",
 363                         path:         "/a/b/c/d.html",
 364                         expectStatus: http.StatusOK,
 365                         expectBodyRe: "<html></html>\n",
 366                         expectHeader: http.Header{
 367                                 "Content-Type": {"text/html; charset=utf-8"},
 368                         },
 369                 },
 370         } {
 371                 c.Logf("trial %#v", trial)
 372                 handler, err := s.localdb.ContainerLog(s.userctx, arvados.ContainerLogOptions{
 373                         UUID:          s.ctrUUID,
 374                         WebDAVOptions: arvados.WebDAVOptions{Path: trial.path},
 375                 })
 376                 c.Assert(err, check.IsNil)
 377                 c.Assert(handler, check.NotNil)
 378                 r, err := http.NewRequestWithContext(s.userctx, trial.method, "https://controller.example/arvados/v1/containers/"+s.ctrUUID+"/log"+trial.path, nil)
 379                 c.Assert(err, check.IsNil)
 380                 for k := range trial.header {
 381                         r.Header.Set(k, trial.header.Get(k))
 382                 }
 383                 rec := httptest.NewRecorder()
 384                 handler.ServeHTTP(rec, r)
 385                 resp := rec.Result()
 386                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
 387                 for k := range trial.expectHeader {
 388                         c.Check(resp.Header.Get(k), check.Equals, trial.expectHeader.Get(k))
 389                 }
 390                 buf, err := ioutil.ReadAll(resp.Body)
 391                 c.Check(err, check.IsNil)
 392                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
 393         }
 394 }
 395
 396 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 397         c.Logf("connecting to %s", s.gw.Address)
 398         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 399         c.Assert(err, check.IsNil)
 400         c.Assert(sshconn.Conn, check.NotNil)
 401         defer sshconn.Conn.Close()
 402
 403         done := make(chan struct{})
 404         go func() {
 405                 defer close(done)
 406
 407                 // Receive text banner
 408                 buf := make([]byte, 12)
 409                 _, err := io.ReadFull(sshconn.Conn, buf)
 410                 c.Check(err, check.IsNil)
 411                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 412
 413                 // Send text banner
 414                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 415                 c.Check(err, check.IsNil)
 416
 417                 // Receive binary
 418                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 419                 c.Check(err, check.IsNil)
 420
 421                 // If we can get this far into an SSH handshake...
 422                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 423         }()
 424         select {
 425         case <-done:
 426         case <-time.After(time.Second):
 427                 c.Fail()
 428         }
 429         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 430         c.Check(err, check.IsNil)
 431         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 432 }
 433
 434 func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
 435         c.Log("trying with no token")
 436         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
 437         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 438         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 439
 440         c.Log("trying with anonymous token")
 441         ctx = ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
 442         _, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 443         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 444 }
 445
 446 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 447         // no AuthSecret
 448         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 449                 UUID: s.ctrUUID,
 450         })
 451         c.Check(err, check.ErrorMatches, `authentication error`)
 452         c.Check(conn.Conn, check.IsNil)
 453
 454         // bogus AuthSecret
 455         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 456                 UUID:       s.ctrUUID,
 457                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 458         })
 459         c.Check(err, check.ErrorMatches, `authentication error`)
 460         c.Check(conn.Conn, check.IsNil)
 461
 462         // good AuthSecret
 463         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 464                 UUID:       s.ctrUUID,
 465                 AuthSecret: s.gw.AuthSecret,
 466         })
 467         c.Check(err, check.IsNil)
 468         c.Check(conn.Conn, check.NotNil)
 469 }
 470
 471 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 472         forceProxyForTest = true
 473         defer func() { forceProxyForTest = false }()
 474         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 475         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 476         s.testConnectThroughTunnel(c, "")
 477 }
 478
 479 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 480         forceProxyForTest = true
 481         defer func() { forceProxyForTest = false }()
 482         // forceInternalURLForTest will not be usable because it isn't
 483         // listed in s.cluster.Services.Controller.InternalURLs
 484         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 485 }
 486
 487 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 488         s.testConnectThroughTunnel(c, "")
 489 }
 490
 491 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
 492         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 493         // Until the tunnel starts up, set gateway_address to a value
 494         // that can't work. We want to ensure the only way we can
 495         // reach the gateway is through the tunnel.
 496         tungw := &crunchrun.Gateway{
 497                 ContainerUUID: s.ctrUUID,
 498                 AuthSecret:    s.gw.AuthSecret,
 499                 Log:           ctxlog.TestLogger(c),
 500                 Target:        crunchrun.GatewayTargetStub{},
 501                 ArvadosClient: s.gw.ArvadosClient,
 502                 UpdateTunnelURL: func(url string) {
 503                         c.Logf("UpdateTunnelURL(%q)", url)
 504                         gwaddr := "tunnel " + url
 505                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 506                                 UUID: s.ctrUUID,
 507                                 Attrs: map[string]interface{}{
 508                                         "gateway_address": gwaddr}})
 509                 },
 510         }
 511         c.Assert(tungw.Start(), check.IsNil)
 512
 513         // We didn't supply an external hostname in the Address field,
 514         // so Start() should assign a local address.
 515         host, _, err := net.SplitHostPort(tungw.Address)
 516         c.Assert(err, check.IsNil)
 517         c.Check(host, check.Equals, "127.0.0.1")
 518
 519         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 520                 UUID: s.ctrUUID,
 521                 Attrs: map[string]interface{}{
 522                         "state": arvados.ContainerStateRunning,
 523                 }})
 524         c.Assert(err, check.IsNil)
 525
 526         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 527                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 528                 c.Assert(err, check.IsNil)
 529                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 530                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 531                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 532                         break
 533                 }
 534         }
 535         return tungw
 536 }
 537
 538 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 539         s.setupGatewayWithTunnel(c)
 540         c.Log("connecting to gateway through tunnel")
 541         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 542         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 543         if expectErrorMatch != "" {
 544                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 545                 return
 546         }
 547         c.Assert(err, check.IsNil)
 548         c.Assert(sshconn.Conn, check.NotNil)
 549         defer sshconn.Conn.Close()
 550
 551         done := make(chan struct{})
 552         go func() {
 553                 defer close(done)
 554
 555                 // Receive text banner
 556                 buf := make([]byte, 12)
 557                 _, err := io.ReadFull(sshconn.Conn, buf)
 558                 c.Check(err, check.IsNil)
 559                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 560
 561                 // Send text banner
 562                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 563                 c.Check(err, check.IsNil)
 564
 565                 // Receive binary
 566                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 567                 c.Check(err, check.IsNil)
 568
 569                 // If we can get this far into an SSH handshake...
 570                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 571         }()
 572         select {
 573         case <-done:
 574         case <-time.After(time.Second):
 575                 c.Fail()
 576         }
 577         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 578         c.Check(err, check.IsNil)
 579         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 580 }