lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/hmac"
  11         "crypto/sha256"
  12         "errors"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "net"
  17         "net/http"
  18         "net/http/httptest"
  19         "net/url"
  20         "os"
  21         "os/exec"
  22         "path/filepath"
  23         "strconv"
  24         "strings"
  25         "sync"
  26         "time"
  27
  28         "git.arvados.org/arvados.git/lib/controller/router"
  29         "git.arvados.org/arvados.git/lib/controller/rpc"
  30         "git.arvados.org/arvados.git/lib/crunchrun"
  31         "git.arvados.org/arvados.git/lib/ctrlctx"
  32         "git.arvados.org/arvados.git/sdk/go/arvados"
  33         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  34         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  35         "git.arvados.org/arvados.git/sdk/go/auth"
  36         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  37         "git.arvados.org/arvados.git/sdk/go/httpserver"
  38         "git.arvados.org/arvados.git/sdk/go/keepclient"
  39         "golang.org/x/crypto/ssh"
  40         check "gopkg.in/check.v1"
  41 )
  42
  43 var _ = check.Suite(&ContainerGatewaySuite{})
  44
  45 type ContainerGatewaySuite struct {
  46         localdbSuite
  47         containerServices []*httpserver.Server
  48         reqCreateOptions  arvados.CreateOptions
  49         reqUUID           string
  50         ctrUUID           string
  51         srv               *httptest.Server
  52         gw                *crunchrun.Gateway
  53 }
  54
  55 func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
  56         s.localdbSuite.SetUpSuite(c)
  57
  58         // Set up 10 http servers to play the role of services running
  59         // inside a container. (crunchrun.GatewayTargetStub will allow
  60         // our crunchrun.Gateway to connect to them directly on
  61         // localhost, rather than actually running them inside a
  62         // container.)
  63         for i := 0; i < 10; i++ {
  64                 srv := &httpserver.Server{
  65                         Addr: ":0",
  66                         Server: http.Server{
  67                                 Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  68                                         w.WriteHeader(http.StatusOK)
  69                                         body := fmt.Sprintf("handled %s %s with Host %s", r.Method, r.URL.String(), r.Host)
  70                                         c.Logf("%s", body)
  71                                         w.Write([]byte(body))
  72                                 }),
  73                         },
  74                 }
  75                 srv.Start()
  76                 s.containerServices = append(s.containerServices, srv)
  77         }
  78
  79         // s.containerServices[0] will be unlisted
  80         // s.containerServices[1] will be listed with access=public
  81         // s.containerServices[2,...] will be listed with access=private
  82         publishedPorts := make(map[string]arvados.PublishedPort)
  83         for i, srv := range s.containerServices {
  84                 access := arvados.PublishedPortAccessPrivate
  85                 _, port, _ := net.SplitHostPort(srv.Addr)
  86                 if i == 1 {
  87                         access = arvados.PublishedPortAccessPublic
  88                 }
  89                 if i > 0 {
  90                         publishedPorts[port] = arvados.PublishedPort{
  91                                 Access: access,
  92                                 Label:  "port " + port,
  93                         }
  94                 }
  95         }
  96
  97         s.reqCreateOptions = arvados.CreateOptions{
  98                 Attrs: map[string]interface{}{
  99                         "command":             []string{"echo", time.Now().Format(time.RFC3339Nano)},
 100                         "container_count_max": 1,
 101                         "container_image":     "arvados/apitestfixture:latest",
 102                         "cwd":                 "/tmp",
 103                         "environment":         map[string]string{},
 104                         "output_path":         "/out",
 105                         "priority":            1,
 106                         "state":               arvados.ContainerRequestStateCommitted,
 107                         "mounts": map[string]interface{}{
 108                                 "/out": map[string]interface{}{
 109                                         "kind":     "tmp",
 110                                         "capacity": 1000000,
 111                                 },
 112                         },
 113                         "runtime_constraints": map[string]interface{}{
 114                                 "vcpus": 1,
 115                                 "ram":   2,
 116                         },
 117                         "published_ports": publishedPorts}}
 118 }
 119
 120 func (s *ContainerGatewaySuite) TearDownSuite(c *check.C) {
 121         for _, srv := range s.containerServices {
 122                 go srv.Close()
 123         }
 124         s.containerServices = nil
 125         s.localdbSuite.TearDownSuite(c)
 126 }
 127
 128 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
 129         s.localdbSuite.SetUpTest(c)
 130
 131         cr, err := s.localdb.ContainerRequestCreate(s.userctx, s.reqCreateOptions)
 132         c.Assert(err, check.IsNil)
 133         s.reqUUID = cr.UUID
 134         s.ctrUUID = cr.ContainerUUID
 135
 136         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
 137         fmt.Fprint(h, s.ctrUUID)
 138         authKey := fmt.Sprintf("%x", h.Sum(nil))
 139
 140         rtr := router.New(s.localdb, router.Config{
 141                 ContainerWebServicesURL: arvados.URL{Host: "*.containers.example.com"},
 142         })
 143         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
 144         s.srv.StartTLS()
 145         // the test setup doesn't use lib/service so
 146         // service.URLFromContext() returns nothing -- instead, this
 147         // is how we advertise our internal URL and enable
 148         // proxy-to-other-controller mode,
 149         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
 150         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 151         ac := &arvados.Client{
 152                 APIHost:   s.srv.Listener.Addr().String(),
 153                 AuthToken: arvadostest.SystemRootToken,
 154                 Insecure:  true,
 155         }
 156         s.gw = &crunchrun.Gateway{
 157                 ContainerUUID: s.ctrUUID,
 158                 AuthSecret:    authKey,
 159                 Address:       "localhost:0",
 160                 Log:           ctxlog.TestLogger(c),
 161                 Target:        crunchrun.GatewayTargetStub{},
 162                 ArvadosClient: ac,
 163         }
 164         c.Assert(s.gw.Start(), check.IsNil)
 165
 166         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 167         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 168                 UUID: s.ctrUUID,
 169                 Attrs: map[string]interface{}{
 170                         "state": arvados.ContainerStateLocked}})
 171         c.Assert(err, check.IsNil)
 172         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 173                 UUID: s.ctrUUID,
 174                 Attrs: map[string]interface{}{
 175                         "state":           arvados.ContainerStateRunning,
 176                         "gateway_address": s.gw.Address}})
 177         c.Assert(err, check.IsNil)
 178
 179         s.cluster.Containers.ShellAccess.Admin = true
 180         s.cluster.Containers.ShellAccess.User = true
 181         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
 182         c.Check(err, check.IsNil)
 183 }
 184
 185 func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
 186         forceProxyForTest = false
 187         if s.reqUUID != "" {
 188                 _, err := s.localdb.ContainerRequestDelete(s.userctx, arvados.DeleteOptions{UUID: s.reqUUID})
 189                 c.Check(err, check.IsNil)
 190         }
 191         if s.srv != nil {
 192                 s.srv.Close()
 193                 s.srv = nil
 194         }
 195         s.localdbSuite.TearDownTest(c)
 196 }
 197
 198 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 199         for _, trial := range []struct {
 200                 configAdmin bool
 201                 configUser  bool
 202                 sendToken   string
 203                 errorCode   int
 204         }{
 205                 {true, true, arvadostest.ActiveTokenV2, 0},
 206                 {true, false, arvadostest.ActiveTokenV2, 503},
 207                 {false, true, arvadostest.ActiveTokenV2, 0},
 208                 {false, false, arvadostest.ActiveTokenV2, 503},
 209                 {true, true, arvadostest.AdminToken, 0},
 210                 {true, false, arvadostest.AdminToken, 0},
 211                 {false, true, arvadostest.AdminToken, 403},
 212                 {false, false, arvadostest.AdminToken, 503},
 213         } {
 214                 c.Logf("trial %#v", trial)
 215                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 216                 s.cluster.Containers.ShellAccess.User = trial.configUser
 217                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 218                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 219                 if trial.errorCode == 0 {
 220                         if !c.Check(err, check.IsNil) {
 221                                 continue
 222                         }
 223                         if !c.Check(sshconn.Conn, check.NotNil) {
 224                                 continue
 225                         }
 226                         sshconn.Conn.Close()
 227                 } else {
 228                         c.Check(err, check.NotNil)
 229                         err, ok := err.(interface{ HTTPStatus() int })
 230                         if c.Check(ok, check.Equals, true) {
 231                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 232                         }
 233                 }
 234         }
 235 }
 236
 237 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 238         // Set up servers on a few TCP ports
 239         var addrs []string
 240         for i := 0; i < 3; i++ {
 241                 ln, err := net.Listen("tcp", ":0")
 242                 c.Assert(err, check.IsNil)
 243                 defer ln.Close()
 244                 addrs = append(addrs, ln.Addr().String())
 245                 go func() {
 246                         for {
 247                                 conn, err := ln.Accept()
 248                                 if err != nil {
 249                                         return
 250                                 }
 251                                 var gotAddr string
 252                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 253                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 254                                 if gotAddr == ln.Addr().String() {
 255                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 256                                 }
 257                                 conn.Close()
 258                         }
 259                 }()
 260         }
 261
 262         c.Logf("connecting to %s", s.gw.Address)
 263         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 264         c.Assert(err, check.IsNil)
 265         c.Assert(sshconn.Conn, check.NotNil)
 266         defer sshconn.Conn.Close()
 267         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 268                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 269         })
 270         c.Assert(err, check.IsNil)
 271         client := ssh.NewClient(conn, chans, reqs)
 272         for _, expectAddr := range addrs {
 273                 _, port, err := net.SplitHostPort(expectAddr)
 274                 c.Assert(err, check.IsNil)
 275
 276                 c.Logf("trying foo:%s", port)
 277                 {
 278                         conn, err := client.Dial("tcp", "foo:"+port)
 279                         c.Assert(err, check.IsNil)
 280                         conn.SetDeadline(time.Now().Add(time.Second))
 281                         buf, err := ioutil.ReadAll(conn)
 282                         c.Check(err, check.IsNil)
 283                         c.Check(string(buf), check.Equals, "")
 284                 }
 285
 286                 c.Logf("trying localhost:%s", port)
 287                 {
 288                         conn, err := client.Dial("tcp", "localhost:"+port)
 289                         c.Assert(err, check.IsNil)
 290                         conn.SetDeadline(time.Now().Add(time.Second))
 291                         conn.Write([]byte(expectAddr + "\n"))
 292                         var gotAddr string
 293                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 294                         c.Check(gotAddr, check.Equals, expectAddr)
 295                 }
 296         }
 297 }
 298
 299 // Connect to crunch-run container gateway directly, using container
 300 // UUID.
 301 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Direct(c *check.C) {
 302         s.testContainerHTTPProxy(c, s.ctrUUID)
 303 }
 304
 305 // Connect to crunch-run container gateway directly, using container
 306 // request UUID.
 307 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Direct_ContainerRequestUUID(c *check.C) {
 308         s.testContainerHTTPProxy(c, s.reqUUID)
 309 }
 310
 311 // Connect through a tunnel terminated at this controller process.
 312 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Tunnel(c *check.C) {
 313         s.gw = s.setupGatewayWithTunnel(c)
 314         s.testContainerHTTPProxy(c, s.ctrUUID)
 315 }
 316
 317 // Connect through a tunnel terminated at a different controller
 318 // process.
 319 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ProxyTunnel(c *check.C) {
 320         forceProxyForTest = true
 321         s.gw = s.setupGatewayWithTunnel(c)
 322         s.testContainerHTTPProxy(c, s.ctrUUID)
 323 }
 324
 325 func (s *ContainerGatewaySuite) testContainerHTTPProxy(c *check.C, targetUUID string) {
 326         testMethods := []string{"GET", "POST", "PATCH", "OPTIONS", "DELETE"}
 327
 328         var wg sync.WaitGroup
 329         for idx, srv := range s.containerServices {
 330                 idx, srv := idx, srv
 331                 wg.Add(1)
 332                 go func() {
 333                         defer wg.Done()
 334                         c.Logf("sending request to %s via %s", srv.Addr, s.gw.Address)
 335                         method := testMethods[idx%len(testMethods)]
 336                         _, port, err := net.SplitHostPort(srv.Addr)
 337                         c.Assert(err, check.IsNil)
 338                         vhost := targetUUID + "-" + port + ".containers.example.com"
 339                         req, err := http.NewRequest(method, "https://"+vhost+"/via-"+s.gw.Address, nil)
 340                         c.Assert(err, check.IsNil)
 341                         // Token is already passed to
 342                         // ContainerHTTPProxy() call in s.userctx, but
 343                         // we also need to add an auth cookie to the
 344                         // http request: if the request gets passed
 345                         // through http (see forceProxyForTest), the
 346                         // target router will start with a fresh
 347                         // context and load tokens from the request.
 348                         req.AddCookie(&http.Cookie{
 349                                 Name:  "arvados_api_token",
 350                                 Value: auth.EncodeTokenCookie([]byte(arvadostest.ActiveTokenV2)),
 351                         })
 352                         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 353                                 Target:  fmt.Sprintf("%s-%s", targetUUID, port),
 354                                 Request: req,
 355                         })
 356                         c.Assert(err, check.IsNil)
 357                         rw := httptest.NewRecorder()
 358                         handler.ServeHTTP(rw, req)
 359                         resp := rw.Result()
 360                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 361                         body, err := io.ReadAll(resp.Body)
 362                         c.Assert(err, check.IsNil)
 363                         c.Check(string(body), check.Matches, `handled `+method+` /via-.* with Host \Q`+vhost+`\E`)
 364                 }()
 365         }
 366         wg.Wait()
 367 }
 368
 369 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_NoToken_Unlisted(c *check.C) {
 370         s.testContainerHTTPProxyError(c, 0, "", http.StatusUnauthorized)
 371 }
 372
 373 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_NoToken_Private(c *check.C) {
 374         s.testContainerHTTPProxyError(c, 2, "", http.StatusUnauthorized)
 375 }
 376
 377 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_InvalidToken(c *check.C) {
 378         s.testContainerHTTPProxyError(c, 0, arvadostest.ActiveTokenV2+"bogus", http.StatusUnauthorized)
 379 }
 380
 381 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_AnonymousToken_Unlisted(c *check.C) {
 382         s.testContainerHTTPProxyError(c, 0, arvadostest.AnonymousToken, http.StatusNotFound)
 383 }
 384
 385 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_AnonymousToken_Private(c *check.C) {
 386         s.testContainerHTTPProxyError(c, 2, arvadostest.AnonymousToken, http.StatusNotFound)
 387 }
 388
 389 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_CRsDifferentUsers(c *check.C) {
 390         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 391         cr, err := s.localdb.ContainerRequestCreate(rootctx, s.reqCreateOptions)
 392         defer s.localdb.ContainerRequestDelete(rootctx, arvados.DeleteOptions{UUID: cr.UUID})
 393         c.Assert(err, check.IsNil)
 394         c.Assert(cr.ContainerUUID, check.Equals, s.ctrUUID)
 395         s.testContainerHTTPProxyError(c, 0, arvadostest.ActiveTokenV2, http.StatusForbidden)
 396 }
 397
 398 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_ContainerNotReadable(c *check.C) {
 399         s.testContainerHTTPProxyError(c, 0, arvadostest.SpectatorToken, http.StatusNotFound)
 400 }
 401
 402 func (s *ContainerGatewaySuite) testContainerHTTPProxyError(c *check.C, svcIdx int, token string, expectCode int) {
 403         _, svcPort, err := net.SplitHostPort(s.containerServices[svcIdx].Addr)
 404         c.Assert(err, check.IsNil)
 405         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, token)
 406         vhost := s.ctrUUID + "-" + svcPort + ".containers.example.com"
 407         req, err := http.NewRequest("GET", "https://"+vhost+"/via-"+s.gw.Address, nil)
 408         c.Assert(err, check.IsNil)
 409         _, err = s.localdb.ContainerHTTPProxy(ctx, arvados.ContainerHTTPProxyOptions{
 410                 Target:  fmt.Sprintf("%s-%s", s.ctrUUID, svcPort),
 411                 Request: req,
 412         })
 413         c.Check(err, check.NotNil)
 414         var se httpserver.HTTPStatusError
 415         c.Assert(errors.As(err, &se), check.Equals, true)
 416         c.Check(se.HTTPStatus(), check.Equals, expectCode)
 417 }
 418
 419 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_CookieAuth(c *check.C) {
 420         s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "GET", "/foobar")
 421 }
 422
 423 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_CookieAuth_POST(c *check.C) {
 424         s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "POST", "/foobar")
 425 }
 426
 427 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth(c *check.C) {
 428         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 429 }
 430
 431 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_Tunnel(c *check.C) {
 432         s.gw = s.setupGatewayWithTunnel(c)
 433         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 434 }
 435
 436 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_ProxyTunnel(c *check.C) {
 437         forceProxyForTest = true
 438         s.gw = s.setupGatewayWithTunnel(c)
 439         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 440 }
 441
 442 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Anonymous(c *check.C) {
 443         s.testContainerHTTPProxyUsingCurl(c, 1, "", "GET", "/foobar")
 444 }
 445
 446 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Anonymous_OPTIONS(c *check.C) {
 447         s.testContainerHTTPProxyUsingCurl(c, 1, "", "OPTIONS", "/foobar")
 448 }
 449
 450 // Check other query parameters are preserved in the
 451 // redirect-with-cookie.
 452 //
 453 // Note the original request has "?baz&baz&..." and this changes to
 454 // "?baz=&baz=&..." in the redirect location.  We trust the target
 455 // service won't be sensitive to this difference.
 456 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_PreserveQuery(c *check.C) {
 457         body := s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?baz&baz&arvados_api_token="+arvadostest.ActiveTokenV2+"&waz=quux")
 458         c.Check(body, check.Matches, `handled GET /foobar\?baz=&baz=&waz=quux with Host `+s.ctrUUID+`.*`)
 459 }
 460
 461 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Patch(c *check.C) {
 462         body := s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "PATCH", "/foobar")
 463         c.Check(body, check.Matches, `handled PATCH /foobar with Host `+s.ctrUUID+`.*`)
 464 }
 465
 466 func (s *ContainerGatewaySuite) testContainerHTTPProxyUsingCurl(c *check.C, svcIdx int, cookietoken, method, path string) string {
 467         _, svcPort, err := net.SplitHostPort(s.containerServices[svcIdx].Addr)
 468         c.Assert(err, check.IsNil)
 469
 470         vhost, err := url.Parse(s.srv.URL)
 471         c.Assert(err, check.IsNil)
 472         controllerHost := vhost.Host
 473         vhost.Host = s.ctrUUID + "-" + svcPort + ".containers.example.com"
 474         target, err := vhost.Parse(path)
 475         c.Assert(err, check.IsNil)
 476
 477         tempdir, err := ioutil.TempDir("", "localdb-test-")
 478         c.Assert(err, check.IsNil)
 479         defer os.RemoveAll(tempdir)
 480
 481         cmd := exec.Command("curl")
 482         if cookietoken != "" {
 483                 cmd.Args = append(cmd.Args, "--cookie", "arvados_api_token="+string(auth.EncodeTokenCookie([]byte(cookietoken))))
 484         } else {
 485                 cmd.Args = append(cmd.Args, "--cookie-jar", filepath.Join(tempdir, "cookie.jar"))
 486         }
 487         if method != "GET" {
 488                 cmd.Args = append(cmd.Args, "--request", method)
 489         }
 490         cmd.Args = append(cmd.Args, "--silent", "--insecure", "--location", "--connect-to", vhost.Hostname()+":443:"+controllerHost, target.String())
 491         cmd.Dir = tempdir
 492         stdout, err := cmd.StdoutPipe()
 493         c.Assert(err, check.Equals, nil)
 494         cmd.Stderr = cmd.Stdout
 495         c.Logf("cmd: %v", cmd.Args)
 496         go cmd.Start()
 497
 498         var buf bytes.Buffer
 499         _, err = io.Copy(&buf, stdout)
 500         c.Check(err, check.Equals, nil)
 501         err = cmd.Wait()
 502         c.Check(err, check.Equals, nil)
 503         c.Check(buf.String(), check.Matches, `handled `+method+` /.*`)
 504         return buf.String()
 505 }
 506
 507 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_PublishedPortByName_ProxyTunnel(c *check.C) {
 508         forceProxyForTest = true
 509         s.gw = s.setupGatewayWithTunnel(c)
 510         s.testContainerHTTPProxy_PublishedPortByName(c)
 511 }
 512
 513 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_PublishedPortByName(c *check.C) {
 514         s.testContainerHTTPProxy_PublishedPortByName(c)
 515 }
 516
 517 func (s *ContainerGatewaySuite) testContainerHTTPProxy_PublishedPortByName(c *check.C) {
 518         srv := s.containerServices[1]
 519         _, port, _ := net.SplitHostPort(srv.Addr)
 520         portnum, err := strconv.Atoi(port)
 521         c.Assert(err, check.IsNil)
 522         namelink, err := s.localdb.LinkCreate(s.userctx, arvados.CreateOptions{
 523                 Attrs: map[string]interface{}{
 524                         "link_class": "published_port",
 525                         "name":       "warthogfacedbuffoon",
 526                         "head_uuid":  s.reqUUID,
 527                         "properties": map[string]interface{}{
 528                                 "port": portnum}}})
 529         c.Assert(err, check.IsNil)
 530         defer s.localdb.LinkDelete(s.userctx, arvados.DeleteOptions{UUID: namelink.UUID})
 531
 532         vhost := namelink.Name + ".containers.example.com"
 533         req, err := http.NewRequest("METHOD", "https://"+vhost+"/path", nil)
 534         c.Assert(err, check.IsNil)
 535         // Token is already passed to ContainerHTTPProxy() call in
 536         // s.userctx, but we also need to add an auth cookie to the
 537         // http request: if the request gets passed through http (see
 538         // forceProxyForTest), the target router will start with a
 539         // fresh context and load tokens from the request.
 540         req.AddCookie(&http.Cookie{
 541                 Name:  "arvados_api_token",
 542                 Value: auth.EncodeTokenCookie([]byte(arvadostest.ActiveTokenV2)),
 543         })
 544         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 545                 Target:  namelink.Name,
 546                 Request: req,
 547         })
 548         c.Assert(err, check.IsNil)
 549         rw := httptest.NewRecorder()
 550         handler.ServeHTTP(rw, req)
 551         resp := rw.Result()
 552         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 553         body, err := io.ReadAll(resp.Body)
 554         c.Assert(err, check.IsNil)
 555         c.Check(string(body), check.Matches, `handled METHOD /path with Host \Q`+vhost+`\E`)
 556 }
 557
 558 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
 559         files := map[string]string{
 560                 "stderr.txt":   "hello world\n",
 561                 "a/b/c/d.html": "<html></html>\n",
 562         }
 563         client := arvados.NewClientFromEnv()
 564         ac, err := arvadosclient.New(client)
 565         c.Assert(err, check.IsNil)
 566         kc, err := keepclient.MakeKeepClient(ac)
 567         c.Assert(err, check.IsNil)
 568         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 569         c.Assert(err, check.IsNil)
 570         for name, content := range files {
 571                 for i, ch := range name {
 572                         if ch == '/' {
 573                                 err := cfs.Mkdir("/"+name[:i], 0777)
 574                                 c.Assert(err, check.IsNil)
 575                         }
 576                 }
 577                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 578                 c.Assert(err, check.IsNil)
 579                 f.Write([]byte(content))
 580                 err = f.Close()
 581                 c.Assert(err, check.IsNil)
 582         }
 583         cfs.Sync()
 584         s.gw.LogCollection = cfs
 585 }
 586
 587 func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
 588         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 589         txt, err := s.gw.LogCollection.MarshalManifest(".")
 590         c.Assert(err, check.IsNil)
 591         coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 592                 Attrs: map[string]interface{}{
 593                         "manifest_text": txt,
 594                 }})
 595         c.Assert(err, check.IsNil)
 596         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 597                 UUID: s.ctrUUID,
 598                 Attrs: map[string]interface{}{
 599                         "state":     arvados.ContainerStateComplete,
 600                         "exit_code": 0,
 601                         "log":       coll.PortableDataHash,
 602                 }})
 603         c.Assert(err, check.IsNil)
 604         updatedReq, err := s.localdb.ContainerRequestGet(rootctx, arvados.GetOptions{UUID: s.reqUUID})
 605         c.Assert(err, check.IsNil)
 606         c.Logf("container request log UUID is %s", updatedReq.LogUUID)
 607         crLog, err := s.localdb.CollectionGet(rootctx, arvados.GetOptions{UUID: updatedReq.LogUUID, Select: []string{"manifest_text"}})
 608         c.Assert(err, check.IsNil)
 609         c.Logf("collection log manifest:\n%s", crLog.ManifestText)
 610         // Ensure localdb can't circumvent the keep-web proxy test by
 611         // getting content from the container gateway.
 612         s.gw.LogCollection = nil
 613 }
 614
 615 func (s *ContainerGatewaySuite) TestContainerRequestLogViaTunnel(c *check.C) {
 616         forceProxyForTest = true
 617         s.gw = s.setupGatewayWithTunnel(c)
 618         s.setupLogCollection(c)
 619
 620         for _, broken := range []bool{false, true} {
 621                 c.Logf("broken=%v", broken)
 622
 623                 if broken {
 624                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 625                 }
 626
 627                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID+"/stderr.txt", nil)
 628                 c.Assert(err, check.IsNil)
 629                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 630                 handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
 631                         UUID: s.reqUUID,
 632                         WebDAVOptions: arvados.WebDAVOptions{
 633                                 Method: "GET",
 634                                 Header: r.Header,
 635                                 Path:   "/" + s.ctrUUID + "/stderr.txt",
 636                         },
 637                 })
 638                 if broken {
 639                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 640                         continue
 641                 }
 642                 c.Check(err, check.IsNil)
 643                 c.Assert(handler, check.NotNil)
 644                 rec := httptest.NewRecorder()
 645                 handler.ServeHTTP(rec, r)
 646                 resp := rec.Result()
 647                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 648                 buf, err := ioutil.ReadAll(resp.Body)
 649                 c.Check(err, check.IsNil)
 650                 c.Check(string(buf), check.Equals, "hello world\n")
 651         }
 652 }
 653
 654 func (s *ContainerGatewaySuite) TestContainerRequestLogViaGateway(c *check.C) {
 655         s.setupLogCollection(c)
 656         s.testContainerRequestLog(c)
 657 }
 658
 659 func (s *ContainerGatewaySuite) TestContainerRequestLogViaKeepWeb(c *check.C) {
 660         s.setupLogCollection(c)
 661         s.saveLogAndCloseGateway(c)
 662         s.testContainerRequestLog(c)
 663 }
 664
 665 func (s *ContainerGatewaySuite) testContainerRequestLog(c *check.C) {
 666         for _, trial := range []struct {
 667                 method          string
 668                 path            string
 669                 header          http.Header
 670                 unauthenticated bool
 671                 expectStatus    int
 672                 expectBodyRe    string
 673                 expectHeader    http.Header
 674         }{
 675                 {
 676                         method:       "GET",
 677                         path:         s.ctrUUID + "/stderr.txt",
 678                         expectStatus: http.StatusOK,
 679                         expectBodyRe: "hello world\n",
 680                         expectHeader: http.Header{
 681                                 "Content-Type": {"text/plain; charset=utf-8"},
 682                         },
 683                 },
 684                 {
 685                         method: "GET",
 686                         path:   s.ctrUUID + "/stderr.txt",
 687                         header: http.Header{
 688                                 "Range": {"bytes=-6"},
 689                         },
 690                         expectStatus: http.StatusPartialContent,
 691                         expectBodyRe: "world\n",
 692                         expectHeader: http.Header{
 693                                 "Content-Type":  {"text/plain; charset=utf-8"},
 694                                 "Content-Range": {"bytes 6-11/12"},
 695                         },
 696                 },
 697                 {
 698                         method:       "OPTIONS",
 699                         path:         s.ctrUUID + "/stderr.txt",
 700                         expectStatus: http.StatusOK,
 701                         expectBodyRe: "",
 702                         expectHeader: http.Header{
 703                                 "Dav":   {"1, 2"},
 704                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 705                         },
 706                 },
 707                 {
 708                         method:          "OPTIONS",
 709                         path:            s.ctrUUID + "/stderr.txt",
 710                         unauthenticated: true,
 711                         header: http.Header{
 712                                 "Access-Control-Request-Method": {"POST"},
 713                         },
 714                         expectStatus: http.StatusOK,
 715                         expectBodyRe: "",
 716                         expectHeader: http.Header{
 717                                 "Access-Control-Allow-Headers": {"Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control"},
 718                                 "Access-Control-Allow-Methods": {"COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK"},
 719                                 "Access-Control-Allow-Origin":  {"*"},
 720                                 "Access-Control-Max-Age":       {"86400"},
 721                         },
 722                 },
 723                 {
 724                         method:       "PROPFIND",
 725                         path:         s.ctrUUID + "/",
 726                         expectStatus: http.StatusMultiStatus,
 727                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 728                         expectHeader: http.Header{
 729                                 "Content-Type": {"text/xml; charset=utf-8"},
 730                         },
 731                 },
 732                 {
 733                         method:       "PROPFIND",
 734                         path:         s.ctrUUID,
 735                         expectStatus: http.StatusMultiStatus,
 736                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 737                         expectHeader: http.Header{
 738                                 "Content-Type": {"text/xml; charset=utf-8"},
 739                         },
 740                 },
 741                 {
 742                         method:       "PROPFIND",
 743                         path:         s.ctrUUID + "/a/b/c/",
 744                         expectStatus: http.StatusMultiStatus,
 745                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*>\n?`,
 746                         expectHeader: http.Header{
 747                                 "Content-Type": {"text/xml; charset=utf-8"},
 748                         },
 749                 },
 750                 {
 751                         method:       "GET",
 752                         path:         s.ctrUUID + "/a/b/c/d.html",
 753                         expectStatus: http.StatusOK,
 754                         expectBodyRe: "<html></html>\n",
 755                         expectHeader: http.Header{
 756                                 "Content-Type": {"text/html; charset=utf-8"},
 757                         },
 758                 },
 759         } {
 760                 c.Logf("trial %#v", trial)
 761                 ctx := s.userctx
 762                 if trial.unauthenticated {
 763                         ctx = auth.NewContext(context.Background(), auth.CredentialsFromRequest(&http.Request{URL: &url.URL{}, Header: http.Header{}}))
 764                 }
 765                 r, err := http.NewRequestWithContext(ctx, trial.method, "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+trial.path, nil)
 766                 c.Assert(err, check.IsNil)
 767                 for k := range trial.header {
 768                         r.Header.Set(k, trial.header.Get(k))
 769                 }
 770                 handler, err := s.localdb.ContainerRequestLog(ctx, arvados.ContainerLogOptions{
 771                         UUID: s.reqUUID,
 772                         WebDAVOptions: arvados.WebDAVOptions{
 773                                 Method: trial.method,
 774                                 Header: r.Header,
 775                                 Path:   "/" + trial.path,
 776                         },
 777                 })
 778                 c.Assert(err, check.IsNil)
 779                 c.Assert(handler, check.NotNil)
 780                 rec := httptest.NewRecorder()
 781                 handler.ServeHTTP(rec, r)
 782                 resp := rec.Result()
 783                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
 784                 for k := range trial.expectHeader {
 785                         c.Check(resp.Header[k], check.DeepEquals, trial.expectHeader[k])
 786                 }
 787                 buf, err := ioutil.ReadAll(resp.Body)
 788                 c.Check(err, check.IsNil)
 789                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
 790         }
 791 }
 792
 793 func (s *ContainerGatewaySuite) TestContainerRequestLogViaCadaver(c *check.C) {
 794         s.setupLogCollection(c)
 795
 796         out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "ls")
 797         c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
 798         c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
 799
 800         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
 801         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 802
 803         s.saveLogAndCloseGateway(c)
 804
 805         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
 806         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 807 }
 808
 809 func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
 810         // Replace s.srv with an HTTP server, otherwise cadaver will
 811         // just fail on TLS cert verification.
 812         s.srv.Close()
 813         rtr := router.New(s.localdb, router.Config{})
 814         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
 815         s.srv.Start()
 816
 817         tempdir, err := ioutil.TempDir("", "localdb-test-")
 818         c.Assert(err, check.IsNil)
 819         defer os.RemoveAll(tempdir)
 820
 821         cmd := exec.Command("cadaver", s.srv.URL+path)
 822         if password != "" {
 823                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 824                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 825                 c.Assert(err, check.IsNil)
 826                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 827                 c.Assert(err, check.IsNil)
 828                 c.Assert(f.Close(), check.IsNil)
 829         }
 830         cmd.Stdin = bytes.NewBufferString(stdin)
 831         cmd.Dir = tempdir
 832         stdout, err := cmd.StdoutPipe()
 833         c.Assert(err, check.Equals, nil)
 834         cmd.Stderr = cmd.Stdout
 835         c.Logf("cmd: %v", cmd.Args)
 836         go cmd.Start()
 837
 838         var buf bytes.Buffer
 839         _, err = io.Copy(&buf, stdout)
 840         c.Check(err, check.Equals, nil)
 841         err = cmd.Wait()
 842         c.Check(err, check.Equals, nil)
 843         return buf.String()
 844 }
 845
 846 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 847         c.Logf("connecting to %s", s.gw.Address)
 848         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 849         c.Assert(err, check.IsNil)
 850         c.Assert(sshconn.Conn, check.NotNil)
 851         defer sshconn.Conn.Close()
 852
 853         done := make(chan struct{})
 854         go func() {
 855                 defer close(done)
 856
 857                 // Receive text banner
 858                 buf := make([]byte, 12)
 859                 _, err := io.ReadFull(sshconn.Conn, buf)
 860                 c.Check(err, check.IsNil)
 861                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 862
 863                 // Send text banner
 864                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 865                 c.Check(err, check.IsNil)
 866
 867                 // Receive binary
 868                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 869                 c.Check(err, check.IsNil)
 870
 871                 // If we can get this far into an SSH handshake...
 872                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 873         }()
 874         select {
 875         case <-done:
 876         case <-time.After(time.Second):
 877                 c.Fail()
 878         }
 879         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 880         c.Check(err, check.IsNil)
 881         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 882 }
 883
 884 func (s *ContainerGatewaySuite) TestConnectFail_NoToken(c *check.C) {
 885         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
 886         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 887         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 888 }
 889
 890 func (s *ContainerGatewaySuite) TestConnectFail_AnonymousToken(c *check.C) {
 891         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
 892         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 893         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 894 }
 895
 896 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 897         // no AuthSecret
 898         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 899                 UUID: s.ctrUUID,
 900         })
 901         c.Check(err, check.ErrorMatches, `authentication error`)
 902         c.Check(conn.Conn, check.IsNil)
 903
 904         // bogus AuthSecret
 905         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 906                 UUID:       s.ctrUUID,
 907                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 908         })
 909         c.Check(err, check.ErrorMatches, `authentication error`)
 910         c.Check(conn.Conn, check.IsNil)
 911
 912         // good AuthSecret
 913         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 914                 UUID:       s.ctrUUID,
 915                 AuthSecret: s.gw.AuthSecret,
 916         })
 917         c.Check(err, check.IsNil)
 918         c.Check(conn.Conn, check.NotNil)
 919 }
 920
 921 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 922         forceProxyForTest = true
 923         s.testConnectThroughTunnel(c, "")
 924 }
 925
 926 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 927         forceProxyForTest = true
 928         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 929         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 930 }
 931
 932 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 933         s.testConnectThroughTunnel(c, "")
 934 }
 935
 936 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
 937         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 938         // Until the tunnel starts up, set gateway_address to a value
 939         // that can't work. We want to ensure the only way we can
 940         // reach the gateway is through the tunnel.
 941         tungw := &crunchrun.Gateway{
 942                 ContainerUUID: s.ctrUUID,
 943                 AuthSecret:    s.gw.AuthSecret,
 944                 Log:           ctxlog.TestLogger(c),
 945                 Target:        crunchrun.GatewayTargetStub{},
 946                 ArvadosClient: s.gw.ArvadosClient,
 947                 UpdateTunnelURL: func(url string) {
 948                         c.Logf("UpdateTunnelURL(%q)", url)
 949                         gwaddr := "tunnel " + url
 950                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 951                                 UUID: s.ctrUUID,
 952                                 Attrs: map[string]interface{}{
 953                                         "gateway_address": gwaddr}})
 954                 },
 955         }
 956         c.Assert(tungw.Start(), check.IsNil)
 957
 958         // We didn't supply an external hostname in the Address field,
 959         // so Start() should assign a local address.
 960         host, _, err := net.SplitHostPort(tungw.Address)
 961         c.Assert(err, check.IsNil)
 962         c.Check(host, check.Equals, "127.0.0.1")
 963
 964         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 965                 UUID: s.ctrUUID,
 966                 Attrs: map[string]interface{}{
 967                         "state": arvados.ContainerStateRunning,
 968                 }})
 969         c.Assert(err, check.IsNil)
 970
 971         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 972                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 973                 c.Assert(err, check.IsNil)
 974                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 975                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 976                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 977                         break
 978                 }
 979         }
 980         return tungw
 981 }
 982
 983 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 984         s.setupGatewayWithTunnel(c)
 985         c.Log("connecting to gateway through tunnel")
 986         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 987         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 988         if expectErrorMatch != "" {
 989                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 990                 return
 991         }
 992         c.Assert(err, check.IsNil)
 993         c.Assert(sshconn.Conn, check.NotNil)
 994         defer sshconn.Conn.Close()
 995
 996         done := make(chan struct{})
 997         go func() {
 998                 defer close(done)
 999
1000                 // Receive text banner
1001                 buf := make([]byte, 12)
1002                 _, err := io.ReadFull(sshconn.Conn, buf)
1003                 c.Check(err, check.IsNil)
1004                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
1005
1006                 // Send text banner
1007                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
1008                 c.Check(err, check.IsNil)
1009
1010                 // Receive binary
1011                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
1012                 c.Check(err, check.IsNil)
1013
1014                 // If we can get this far into an SSH handshake...
1015                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
1016         }()
1017         select {
1018         case <-done:
1019         case <-time.After(time.Second):
1020                 c.Fail()
1021         }
1022         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
1023         c.Check(err, check.IsNil)
1024         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
1025 }