lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bytes"
   9         "crypto/hmac"
  10         "crypto/sha256"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http"
  16         "net/http/httptest"
  17         "net/url"
  18         "os"
  19         "os/exec"
  20         "path/filepath"
  21         "strings"
  22         "time"
  23
  24         "git.arvados.org/arvados.git/lib/controller/router"
  25         "git.arvados.org/arvados.git/lib/controller/rpc"
  26         "git.arvados.org/arvados.git/lib/crunchrun"
  27         "git.arvados.org/arvados.git/lib/ctrlctx"
  28         "git.arvados.org/arvados.git/sdk/go/arvados"
  29         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  30         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  31         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  32         "git.arvados.org/arvados.git/sdk/go/httpserver"
  33         "git.arvados.org/arvados.git/sdk/go/keepclient"
  34         "golang.org/x/crypto/ssh"
  35         check "gopkg.in/check.v1"
  36 )
  37
  38 var _ = check.Suite(&ContainerGatewaySuite{})
  39
  40 type ContainerGatewaySuite struct {
  41         localdbSuite
  42         reqUUID string
  43         ctrUUID string
  44         srv     *httptest.Server
  45         gw      *crunchrun.Gateway
  46 }
  47
  48 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
  49         s.localdbSuite.SetUpTest(c)
  50
  51         cr, err := s.localdb.ContainerRequestCreate(s.userctx, arvados.CreateOptions{
  52                 Attrs: map[string]interface{}{
  53                         "command":             []string{"echo", time.Now().Format(time.RFC3339Nano)},
  54                         "container_count_max": 1,
  55                         "container_image":     "arvados/apitestfixture:latest",
  56                         "cwd":                 "/tmp",
  57                         "environment":         map[string]string{},
  58                         "output_path":         "/out",
  59                         "priority":            1,
  60                         "state":               arvados.ContainerRequestStateCommitted,
  61                         "mounts": map[string]interface{}{
  62                                 "/out": map[string]interface{}{
  63                                         "kind":     "tmp",
  64                                         "capacity": 1000000,
  65                                 },
  66                         },
  67                         "runtime_constraints": map[string]interface{}{
  68                                 "vcpus": 1,
  69                                 "ram":   2,
  70                         }}})
  71         c.Assert(err, check.IsNil)
  72         s.reqUUID = cr.UUID
  73         s.ctrUUID = cr.ContainerUUID
  74
  75         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
  76         fmt.Fprint(h, s.ctrUUID)
  77         authKey := fmt.Sprintf("%x", h.Sum(nil))
  78
  79         rtr := router.New(s.localdb, router.Config{})
  80         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
  81         s.srv.StartTLS()
  82         // the test setup doesn't use lib/service so
  83         // service.URLFromContext() returns nothing -- instead, this
  84         // is how we advertise our internal URL and enable
  85         // proxy-to-other-controller mode,
  86         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
  87         ac := &arvados.Client{
  88                 APIHost:   s.srv.Listener.Addr().String(),
  89                 AuthToken: arvadostest.Dispatch1Token,
  90                 Insecure:  true,
  91         }
  92         s.gw = &crunchrun.Gateway{
  93                 ContainerUUID: s.ctrUUID,
  94                 AuthSecret:    authKey,
  95                 Address:       "localhost:0",
  96                 Log:           ctxlog.TestLogger(c),
  97                 Target:        crunchrun.GatewayTargetStub{},
  98                 ArvadosClient: ac,
  99         }
 100         c.Assert(s.gw.Start(), check.IsNil)
 101
 102         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 103         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 104                 UUID: s.ctrUUID,
 105                 Attrs: map[string]interface{}{
 106                         "state": arvados.ContainerStateLocked}})
 107         c.Assert(err, check.IsNil)
 108         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 109                 UUID: s.ctrUUID,
 110                 Attrs: map[string]interface{}{
 111                         "state":           arvados.ContainerStateRunning,
 112                         "gateway_address": s.gw.Address}})
 113         c.Assert(err, check.IsNil)
 114
 115         s.cluster.Containers.ShellAccess.Admin = true
 116         s.cluster.Containers.ShellAccess.User = true
 117         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
 118         c.Check(err, check.IsNil)
 119 }
 120
 121 func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
 122         s.srv.Close()
 123         s.localdbSuite.TearDownTest(c)
 124 }
 125
 126 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 127         for _, trial := range []struct {
 128                 configAdmin bool
 129                 configUser  bool
 130                 sendToken   string
 131                 errorCode   int
 132         }{
 133                 {true, true, arvadostest.ActiveTokenV2, 0},
 134                 {true, false, arvadostest.ActiveTokenV2, 503},
 135                 {false, true, arvadostest.ActiveTokenV2, 0},
 136                 {false, false, arvadostest.ActiveTokenV2, 503},
 137                 {true, true, arvadostest.AdminToken, 0},
 138                 {true, false, arvadostest.AdminToken, 0},
 139                 {false, true, arvadostest.AdminToken, 403},
 140                 {false, false, arvadostest.AdminToken, 503},
 141         } {
 142                 c.Logf("trial %#v", trial)
 143                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 144                 s.cluster.Containers.ShellAccess.User = trial.configUser
 145                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 146                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 147                 if trial.errorCode == 0 {
 148                         if !c.Check(err, check.IsNil) {
 149                                 continue
 150                         }
 151                         if !c.Check(sshconn.Conn, check.NotNil) {
 152                                 continue
 153                         }
 154                         sshconn.Conn.Close()
 155                 } else {
 156                         c.Check(err, check.NotNil)
 157                         err, ok := err.(interface{ HTTPStatus() int })
 158                         if c.Check(ok, check.Equals, true) {
 159                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 160                         }
 161                 }
 162         }
 163 }
 164
 165 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 166         // Set up servers on a few TCP ports
 167         var addrs []string
 168         for i := 0; i < 3; i++ {
 169                 ln, err := net.Listen("tcp", ":0")
 170                 c.Assert(err, check.IsNil)
 171                 defer ln.Close()
 172                 addrs = append(addrs, ln.Addr().String())
 173                 go func() {
 174                         for {
 175                                 conn, err := ln.Accept()
 176                                 if err != nil {
 177                                         return
 178                                 }
 179                                 var gotAddr string
 180                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 181                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 182                                 if gotAddr == ln.Addr().String() {
 183                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 184                                 }
 185                                 conn.Close()
 186                         }
 187                 }()
 188         }
 189
 190         c.Logf("connecting to %s", s.gw.Address)
 191         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 192         c.Assert(err, check.IsNil)
 193         c.Assert(sshconn.Conn, check.NotNil)
 194         defer sshconn.Conn.Close()
 195         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 196                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 197         })
 198         c.Assert(err, check.IsNil)
 199         client := ssh.NewClient(conn, chans, reqs)
 200         for _, expectAddr := range addrs {
 201                 _, port, err := net.SplitHostPort(expectAddr)
 202                 c.Assert(err, check.IsNil)
 203
 204                 c.Logf("trying foo:%s", port)
 205                 {
 206                         conn, err := client.Dial("tcp", "foo:"+port)
 207                         c.Assert(err, check.IsNil)
 208                         conn.SetDeadline(time.Now().Add(time.Second))
 209                         buf, err := ioutil.ReadAll(conn)
 210                         c.Check(err, check.IsNil)
 211                         c.Check(string(buf), check.Equals, "")
 212                 }
 213
 214                 c.Logf("trying localhost:%s", port)
 215                 {
 216                         conn, err := client.Dial("tcp", "localhost:"+port)
 217                         c.Assert(err, check.IsNil)
 218                         conn.SetDeadline(time.Now().Add(time.Second))
 219                         conn.Write([]byte(expectAddr + "\n"))
 220                         var gotAddr string
 221                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 222                         c.Check(gotAddr, check.Equals, expectAddr)
 223                 }
 224         }
 225 }
 226
 227 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
 228         files := map[string]string{
 229                 "stderr.txt":   "hello world\n",
 230                 "a/b/c/d.html": "<html></html>\n",
 231         }
 232         client := arvados.NewClientFromEnv()
 233         ac, err := arvadosclient.New(client)
 234         c.Assert(err, check.IsNil)
 235         kc, err := keepclient.MakeKeepClient(ac)
 236         c.Assert(err, check.IsNil)
 237         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 238         c.Assert(err, check.IsNil)
 239         for name, content := range files {
 240                 for i, ch := range name {
 241                         if ch == '/' {
 242                                 err := cfs.Mkdir("/"+name[:i], 0777)
 243                                 c.Assert(err, check.IsNil)
 244                         }
 245                 }
 246                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 247                 c.Assert(err, check.IsNil)
 248                 f.Write([]byte(content))
 249                 err = f.Close()
 250                 c.Assert(err, check.IsNil)
 251         }
 252         cfs.Sync()
 253         s.gw.LogCollection = cfs
 254 }
 255
 256 func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
 257         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 258         txt, err := s.gw.LogCollection.MarshalManifest(".")
 259         c.Assert(err, check.IsNil)
 260         coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 261                 Attrs: map[string]interface{}{
 262                         "manifest_text": txt,
 263                 }})
 264         c.Assert(err, check.IsNil)
 265         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 266                 UUID: s.ctrUUID,
 267                 Attrs: map[string]interface{}{
 268                         "state":     arvados.ContainerStateComplete,
 269                         "exit_code": 0,
 270                         "log":       coll.PortableDataHash,
 271                 }})
 272         c.Assert(err, check.IsNil)
 273         updatedReq, err := s.localdb.ContainerRequestGet(rootctx, arvados.GetOptions{UUID: s.reqUUID})
 274         c.Assert(err, check.IsNil)
 275         c.Logf("container request log UUID is %s", updatedReq.LogUUID)
 276         crLog, err := s.localdb.CollectionGet(rootctx, arvados.GetOptions{UUID: updatedReq.LogUUID, Select: []string{"manifest_text"}})
 277         c.Assert(err, check.IsNil)
 278         c.Logf("collection log manifest:\n%s", crLog.ManifestText)
 279         // Ensure localdb can't circumvent the keep-web proxy test by
 280         // getting content from the container gateway.
 281         s.gw.LogCollection = nil
 282 }
 283
 284 func (s *ContainerGatewaySuite) TestContainerRequestLogViaTunnel(c *check.C) {
 285         forceProxyForTest = true
 286         defer func() { forceProxyForTest = false }()
 287
 288         s.gw = s.setupGatewayWithTunnel(c)
 289         s.setupLogCollection(c)
 290
 291         for _, broken := range []bool{false, true} {
 292                 c.Logf("broken=%v", broken)
 293
 294                 if broken {
 295                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 296                 } else {
 297                         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 298                         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 299                 }
 300
 301                 handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
 302                         UUID:          s.reqUUID,
 303                         WebDAVOptions: arvados.WebDAVOptions{Path: "/" + s.ctrUUID + "/stderr.txt"},
 304                 })
 305                 if broken {
 306                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 307                         continue
 308                 }
 309                 c.Check(err, check.IsNil)
 310                 c.Assert(handler, check.NotNil)
 311                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID+"/stderr.txt", nil)
 312                 c.Assert(err, check.IsNil)
 313                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 314                 rec := httptest.NewRecorder()
 315                 handler.ServeHTTP(rec, r)
 316                 resp := rec.Result()
 317                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 318                 buf, err := ioutil.ReadAll(resp.Body)
 319                 c.Check(err, check.IsNil)
 320                 c.Check(string(buf), check.Equals, "hello world\n")
 321         }
 322 }
 323
 324 func (s *ContainerGatewaySuite) TestContainerRequestLogViaGateway(c *check.C) {
 325         s.setupLogCollection(c)
 326         s.testContainerRequestLog(c)
 327 }
 328
 329 func (s *ContainerGatewaySuite) TestContainerRequestLogViaKeepWeb(c *check.C) {
 330         s.setupLogCollection(c)
 331         s.saveLogAndCloseGateway(c)
 332         s.testContainerRequestLog(c)
 333 }
 334
 335 func (s *ContainerGatewaySuite) testContainerRequestLog(c *check.C) {
 336         for _, trial := range []struct {
 337                 method       string
 338                 path         string
 339                 header       http.Header
 340                 expectStatus int
 341                 expectBodyRe string
 342                 expectHeader http.Header
 343         }{
 344                 {
 345                         method:       "GET",
 346                         path:         s.ctrUUID + "/stderr.txt",
 347                         expectStatus: http.StatusOK,
 348                         expectBodyRe: "hello world\n",
 349                         expectHeader: http.Header{
 350                                 "Content-Type": {"text/plain; charset=utf-8"},
 351                         },
 352                 },
 353                 {
 354                         method: "GET",
 355                         path:   s.ctrUUID + "/stderr.txt",
 356                         header: http.Header{
 357                                 "Range": {"bytes=-6"},
 358                         },
 359                         expectStatus: http.StatusPartialContent,
 360                         expectBodyRe: "world\n",
 361                         expectHeader: http.Header{
 362                                 "Content-Type":  {"text/plain; charset=utf-8"},
 363                                 "Content-Range": {"bytes 6-11/12"},
 364                         },
 365                 },
 366                 {
 367                         method:       "OPTIONS",
 368                         path:         s.ctrUUID + "/stderr.txt",
 369                         expectStatus: http.StatusOK,
 370                         expectBodyRe: "",
 371                         expectHeader: http.Header{
 372                                 "Dav":   {"1, 2"},
 373                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 374                         },
 375                 },
 376                 {
 377                         method:       "PROPFIND",
 378                         path:         s.ctrUUID + "/",
 379                         expectStatus: http.StatusMultiStatus,
 380                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 381                         expectHeader: http.Header{
 382                                 "Content-Type": {"text/xml; charset=utf-8"},
 383                         },
 384                 },
 385                 {
 386                         method:       "PROPFIND",
 387                         path:         s.ctrUUID,
 388                         expectStatus: http.StatusMultiStatus,
 389                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 390                         expectHeader: http.Header{
 391                                 "Content-Type": {"text/xml; charset=utf-8"},
 392                         },
 393                 },
 394                 {
 395                         method:       "PROPFIND",
 396                         path:         s.ctrUUID + "/a/b/c/",
 397                         expectStatus: http.StatusMultiStatus,
 398                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*>\n?`,
 399                         expectHeader: http.Header{
 400                                 "Content-Type": {"text/xml; charset=utf-8"},
 401                         },
 402                 },
 403                 {
 404                         method:       "GET",
 405                         path:         s.ctrUUID + "/a/b/c/d.html",
 406                         expectStatus: http.StatusOK,
 407                         expectBodyRe: "<html></html>\n",
 408                         expectHeader: http.Header{
 409                                 "Content-Type": {"text/html; charset=utf-8"},
 410                         },
 411                 },
 412         } {
 413                 c.Logf("trial %#v", trial)
 414                 handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
 415                         UUID:          s.reqUUID,
 416                         WebDAVOptions: arvados.WebDAVOptions{Path: "/" + trial.path},
 417                 })
 418                 c.Assert(err, check.IsNil)
 419                 c.Assert(handler, check.NotNil)
 420                 r, err := http.NewRequestWithContext(s.userctx, trial.method, "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+trial.path, nil)
 421                 c.Assert(err, check.IsNil)
 422                 for k := range trial.header {
 423                         r.Header.Set(k, trial.header.Get(k))
 424                 }
 425                 rec := httptest.NewRecorder()
 426                 handler.ServeHTTP(rec, r)
 427                 resp := rec.Result()
 428                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
 429                 for k := range trial.expectHeader {
 430                         c.Check(resp.Header.Get(k), check.Equals, trial.expectHeader.Get(k))
 431                 }
 432                 buf, err := ioutil.ReadAll(resp.Body)
 433                 c.Check(err, check.IsNil)
 434                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
 435         }
 436 }
 437
 438 func (s *ContainerGatewaySuite) TestContainerRequestLogViaCadaver(c *check.C) {
 439         s.setupLogCollection(c)
 440
 441         out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "ls")
 442         c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
 443         c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
 444
 445         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
 446         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 447
 448         s.saveLogAndCloseGateway(c)
 449
 450         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
 451         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 452 }
 453
 454 func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
 455         // Replace s.srv with an HTTP server, otherwise cadaver will
 456         // just fail on TLS cert verification.
 457         s.srv.Close()
 458         rtr := router.New(s.localdb, router.Config{})
 459         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
 460         s.srv.Start()
 461
 462         tempdir, err := ioutil.TempDir("", "localdb-test-")
 463         c.Assert(err, check.IsNil)
 464         defer os.RemoveAll(tempdir)
 465
 466         cmd := exec.Command("cadaver", s.srv.URL+path)
 467         if password != "" {
 468                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 469                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 470                 c.Assert(err, check.IsNil)
 471                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 472                 c.Assert(err, check.IsNil)
 473                 c.Assert(f.Close(), check.IsNil)
 474         }
 475         cmd.Stdin = bytes.NewBufferString(stdin)
 476         cmd.Dir = tempdir
 477         stdout, err := cmd.StdoutPipe()
 478         c.Assert(err, check.Equals, nil)
 479         cmd.Stderr = cmd.Stdout
 480         c.Logf("cmd: %v", cmd.Args)
 481         go cmd.Start()
 482
 483         var buf bytes.Buffer
 484         _, err = io.Copy(&buf, stdout)
 485         c.Check(err, check.Equals, nil)
 486         err = cmd.Wait()
 487         c.Check(err, check.Equals, nil)
 488         return buf.String()
 489 }
 490
 491 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 492         c.Logf("connecting to %s", s.gw.Address)
 493         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 494         c.Assert(err, check.IsNil)
 495         c.Assert(sshconn.Conn, check.NotNil)
 496         defer sshconn.Conn.Close()
 497
 498         done := make(chan struct{})
 499         go func() {
 500                 defer close(done)
 501
 502                 // Receive text banner
 503                 buf := make([]byte, 12)
 504                 _, err := io.ReadFull(sshconn.Conn, buf)
 505                 c.Check(err, check.IsNil)
 506                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 507
 508                 // Send text banner
 509                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 510                 c.Check(err, check.IsNil)
 511
 512                 // Receive binary
 513                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 514                 c.Check(err, check.IsNil)
 515
 516                 // If we can get this far into an SSH handshake...
 517                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 518         }()
 519         select {
 520         case <-done:
 521         case <-time.After(time.Second):
 522                 c.Fail()
 523         }
 524         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 525         c.Check(err, check.IsNil)
 526         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 527 }
 528
 529 func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
 530         c.Log("trying with no token")
 531         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
 532         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 533         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 534
 535         c.Log("trying with anonymous token")
 536         ctx = ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
 537         _, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 538         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 539 }
 540
 541 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 542         // no AuthSecret
 543         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 544                 UUID: s.ctrUUID,
 545         })
 546         c.Check(err, check.ErrorMatches, `authentication error`)
 547         c.Check(conn.Conn, check.IsNil)
 548
 549         // bogus AuthSecret
 550         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 551                 UUID:       s.ctrUUID,
 552                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 553         })
 554         c.Check(err, check.ErrorMatches, `authentication error`)
 555         c.Check(conn.Conn, check.IsNil)
 556
 557         // good AuthSecret
 558         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 559                 UUID:       s.ctrUUID,
 560                 AuthSecret: s.gw.AuthSecret,
 561         })
 562         c.Check(err, check.IsNil)
 563         c.Check(conn.Conn, check.NotNil)
 564 }
 565
 566 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 567         forceProxyForTest = true
 568         defer func() { forceProxyForTest = false }()
 569         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 570         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 571         s.testConnectThroughTunnel(c, "")
 572 }
 573
 574 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 575         forceProxyForTest = true
 576         defer func() { forceProxyForTest = false }()
 577         // forceInternalURLForTest will not be usable because it isn't
 578         // listed in s.cluster.Services.Controller.InternalURLs
 579         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 580 }
 581
 582 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 583         s.testConnectThroughTunnel(c, "")
 584 }
 585
 586 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
 587         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 588         // Until the tunnel starts up, set gateway_address to a value
 589         // that can't work. We want to ensure the only way we can
 590         // reach the gateway is through the tunnel.
 591         tungw := &crunchrun.Gateway{
 592                 ContainerUUID: s.ctrUUID,
 593                 AuthSecret:    s.gw.AuthSecret,
 594                 Log:           ctxlog.TestLogger(c),
 595                 Target:        crunchrun.GatewayTargetStub{},
 596                 ArvadosClient: s.gw.ArvadosClient,
 597                 UpdateTunnelURL: func(url string) {
 598                         c.Logf("UpdateTunnelURL(%q)", url)
 599                         gwaddr := "tunnel " + url
 600                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 601                                 UUID: s.ctrUUID,
 602                                 Attrs: map[string]interface{}{
 603                                         "gateway_address": gwaddr}})
 604                 },
 605         }
 606         c.Assert(tungw.Start(), check.IsNil)
 607
 608         // We didn't supply an external hostname in the Address field,
 609         // so Start() should assign a local address.
 610         host, _, err := net.SplitHostPort(tungw.Address)
 611         c.Assert(err, check.IsNil)
 612         c.Check(host, check.Equals, "127.0.0.1")
 613
 614         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 615                 UUID: s.ctrUUID,
 616                 Attrs: map[string]interface{}{
 617                         "state": arvados.ContainerStateRunning,
 618                 }})
 619         c.Assert(err, check.IsNil)
 620
 621         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 622                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 623                 c.Assert(err, check.IsNil)
 624                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 625                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 626                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 627                         break
 628                 }
 629         }
 630         return tungw
 631 }
 632
 633 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 634         s.setupGatewayWithTunnel(c)
 635         c.Log("connecting to gateway through tunnel")
 636         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 637         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 638         if expectErrorMatch != "" {
 639                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 640                 return
 641         }
 642         c.Assert(err, check.IsNil)
 643         c.Assert(sshconn.Conn, check.NotNil)
 644         defer sshconn.Conn.Close()
 645
 646         done := make(chan struct{})
 647         go func() {
 648                 defer close(done)
 649
 650                 // Receive text banner
 651                 buf := make([]byte, 12)
 652                 _, err := io.ReadFull(sshconn.Conn, buf)
 653                 c.Check(err, check.IsNil)
 654                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 655
 656                 // Send text banner
 657                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 658                 c.Check(err, check.IsNil)
 659
 660                 // Receive binary
 661                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 662                 c.Check(err, check.IsNil)
 663
 664                 // If we can get this far into an SSH handshake...
 665                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 666         }()
 667         select {
 668         case <-done:
 669         case <-time.After(time.Second):
 670                 c.Fail()
 671         }
 672         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 673         c.Check(err, check.IsNil)
 674         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 675 }