lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "crypto/hmac"
   9         "crypto/sha256"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net"
  14         "net/http/httptest"
  15         "net/url"
  16         "strings"
  17         "time"
  18
  19         "git.arvados.org/arvados.git/lib/controller/router"
  20         "git.arvados.org/arvados.git/lib/controller/rpc"
  21         "git.arvados.org/arvados.git/lib/crunchrun"
  22         "git.arvados.org/arvados.git/lib/ctrlctx"
  23         "git.arvados.org/arvados.git/sdk/go/arvados"
  24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "golang.org/x/crypto/ssh"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 var _ = check.Suite(&ContainerGatewaySuite{})
  31
  32 type ContainerGatewaySuite struct {
  33         localdbSuite
  34         ctrUUID string
  35         gw      *crunchrun.Gateway
  36 }
  37
  38 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
  39         s.localdbSuite.SetUpTest(c)
  40
  41         s.ctrUUID = arvadostest.QueuedContainerUUID
  42
  43         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
  44         fmt.Fprint(h, s.ctrUUID)
  45         authKey := fmt.Sprintf("%x", h.Sum(nil))
  46
  47         rtr := router.New(s.localdb, router.Config{})
  48         srv := httptest.NewUnstartedServer(rtr)
  49         srv.StartTLS()
  50         // the test setup doesn't use lib/service so
  51         // service.URLFromContext() returns nothing -- instead, this
  52         // is how we advertise our internal URL and enable
  53         // proxy-to-other-controller mode,
  54         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: srv.Listener.Addr().String()}
  55         ac := &arvados.Client{
  56                 APIHost:   srv.Listener.Addr().String(),
  57                 AuthToken: arvadostest.Dispatch1Token,
  58                 Insecure:  true,
  59         }
  60         s.gw = &crunchrun.Gateway{
  61                 ContainerUUID: s.ctrUUID,
  62                 AuthSecret:    authKey,
  63                 Address:       "localhost:0",
  64                 Log:           ctxlog.TestLogger(c),
  65                 Target:        crunchrun.GatewayTargetStub{},
  66                 ArvadosClient: ac,
  67         }
  68         c.Assert(s.gw.Start(), check.IsNil)
  69         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
  70         // OK if this line fails (because state is already Running
  71         // from a previous test case) as long as the following line
  72         // succeeds:
  73         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  74                 UUID: s.ctrUUID,
  75                 Attrs: map[string]interface{}{
  76                         "state": arvados.ContainerStateLocked}})
  77         _, err := s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  78                 UUID: s.ctrUUID,
  79                 Attrs: map[string]interface{}{
  80                         "state":           arvados.ContainerStateRunning,
  81                         "gateway_address": s.gw.Address}})
  82         c.Assert(err, check.IsNil)
  83
  84         s.cluster.Containers.ShellAccess.Admin = true
  85         s.cluster.Containers.ShellAccess.User = true
  86         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
  87         c.Check(err, check.IsNil)
  88 }
  89
  90 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
  91         for _, trial := range []struct {
  92                 configAdmin bool
  93                 configUser  bool
  94                 sendToken   string
  95                 errorCode   int
  96         }{
  97                 {true, true, arvadostest.ActiveTokenV2, 0},
  98                 {true, false, arvadostest.ActiveTokenV2, 503},
  99                 {false, true, arvadostest.ActiveTokenV2, 0},
 100                 {false, false, arvadostest.ActiveTokenV2, 503},
 101                 {true, true, arvadostest.AdminToken, 0},
 102                 {true, false, arvadostest.AdminToken, 0},
 103                 {false, true, arvadostest.AdminToken, 403},
 104                 {false, false, arvadostest.AdminToken, 503},
 105         } {
 106                 c.Logf("trial %#v", trial)
 107                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 108                 s.cluster.Containers.ShellAccess.User = trial.configUser
 109                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 110                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 111                 if trial.errorCode == 0 {
 112                         if !c.Check(err, check.IsNil) {
 113                                 continue
 114                         }
 115                         if !c.Check(sshconn.Conn, check.NotNil) {
 116                                 continue
 117                         }
 118                         sshconn.Conn.Close()
 119                 } else {
 120                         c.Check(err, check.NotNil)
 121                         err, ok := err.(interface{ HTTPStatus() int })
 122                         if c.Check(ok, check.Equals, true) {
 123                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 124                         }
 125                 }
 126         }
 127 }
 128
 129 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 130         // Set up servers on a few TCP ports
 131         var addrs []string
 132         for i := 0; i < 3; i++ {
 133                 ln, err := net.Listen("tcp", ":0")
 134                 c.Assert(err, check.IsNil)
 135                 defer ln.Close()
 136                 addrs = append(addrs, ln.Addr().String())
 137                 go func() {
 138                         for {
 139                                 conn, err := ln.Accept()
 140                                 if err != nil {
 141                                         return
 142                                 }
 143                                 var gotAddr string
 144                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 145                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 146                                 if gotAddr == ln.Addr().String() {
 147                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 148                                 }
 149                                 conn.Close()
 150                         }
 151                 }()
 152         }
 153
 154         c.Logf("connecting to %s", s.gw.Address)
 155         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 156         c.Assert(err, check.IsNil)
 157         c.Assert(sshconn.Conn, check.NotNil)
 158         defer sshconn.Conn.Close()
 159         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 160                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 161         })
 162         c.Assert(err, check.IsNil)
 163         client := ssh.NewClient(conn, chans, reqs)
 164         for _, expectAddr := range addrs {
 165                 _, port, err := net.SplitHostPort(expectAddr)
 166                 c.Assert(err, check.IsNil)
 167
 168                 c.Logf("trying foo:%s", port)
 169                 {
 170                         conn, err := client.Dial("tcp", "foo:"+port)
 171                         c.Assert(err, check.IsNil)
 172                         conn.SetDeadline(time.Now().Add(time.Second))
 173                         buf, err := ioutil.ReadAll(conn)
 174                         c.Check(err, check.IsNil)
 175                         c.Check(string(buf), check.Equals, "")
 176                 }
 177
 178                 c.Logf("trying localhost:%s", port)
 179                 {
 180                         conn, err := client.Dial("tcp", "localhost:"+port)
 181                         c.Assert(err, check.IsNil)
 182                         conn.SetDeadline(time.Now().Add(time.Second))
 183                         conn.Write([]byte(expectAddr + "\n"))
 184                         var gotAddr string
 185                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 186                         c.Check(gotAddr, check.Equals, expectAddr)
 187                 }
 188         }
 189 }
 190
 191 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 192         c.Logf("connecting to %s", s.gw.Address)
 193         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 194         c.Assert(err, check.IsNil)
 195         c.Assert(sshconn.Conn, check.NotNil)
 196         defer sshconn.Conn.Close()
 197
 198         done := make(chan struct{})
 199         go func() {
 200                 defer close(done)
 201
 202                 // Receive text banner
 203                 buf := make([]byte, 12)
 204                 _, err := io.ReadFull(sshconn.Conn, buf)
 205                 c.Check(err, check.IsNil)
 206                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 207
 208                 // Send text banner
 209                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 210                 c.Check(err, check.IsNil)
 211
 212                 // Receive binary
 213                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 214                 c.Check(err, check.IsNil)
 215
 216                 // If we can get this far into an SSH handshake...
 217                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 218         }()
 219         select {
 220         case <-done:
 221         case <-time.After(time.Second):
 222                 c.Fail()
 223         }
 224         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 225         c.Check(err, check.IsNil)
 226         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 227 }
 228
 229 func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
 230         c.Log("trying with no token")
 231         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
 232         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 233         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 234
 235         c.Log("trying with anonymous token")
 236         ctx = ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
 237         _, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 238         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 239 }
 240
 241 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 242         // no AuthSecret
 243         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 244                 UUID: s.ctrUUID,
 245         })
 246         c.Check(err, check.ErrorMatches, `authentication error`)
 247         c.Check(conn.Conn, check.IsNil)
 248
 249         // bogus AuthSecret
 250         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 251                 UUID:       s.ctrUUID,
 252                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 253         })
 254         c.Check(err, check.ErrorMatches, `authentication error`)
 255         c.Check(conn.Conn, check.IsNil)
 256
 257         // good AuthSecret
 258         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 259                 UUID:       s.ctrUUID,
 260                 AuthSecret: s.gw.AuthSecret,
 261         })
 262         c.Check(err, check.IsNil)
 263         c.Check(conn.Conn, check.NotNil)
 264 }
 265
 266 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 267         forceProxyForTest = true
 268         defer func() { forceProxyForTest = false }()
 269         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 270         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 271         s.testConnectThroughTunnel(c, "")
 272 }
 273
 274 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 275         forceProxyForTest = true
 276         defer func() { forceProxyForTest = false }()
 277         // forceInternalURLForTest shouldn't be used because it isn't
 278         // listed in s.cluster.Services.Controller.InternalURLs
 279         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 280 }
 281
 282 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 283         s.testConnectThroughTunnel(c, "")
 284 }
 285
 286 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 287         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 288         // Until the tunnel starts up, set gateway_address to a value
 289         // that can't work. We want to ensure the only way we can
 290         // reach the gateway is through the tunnel.
 291         tungw := &crunchrun.Gateway{
 292                 ContainerUUID: s.ctrUUID,
 293                 AuthSecret:    s.gw.AuthSecret,
 294                 Log:           ctxlog.TestLogger(c),
 295                 Target:        crunchrun.GatewayTargetStub{},
 296                 ArvadosClient: s.gw.ArvadosClient,
 297                 UpdateTunnelURL: func(url string) {
 298                         c.Logf("UpdateTunnelURL(%q)", url)
 299                         gwaddr := "tunnel " + url
 300                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 301                                 UUID: s.ctrUUID,
 302                                 Attrs: map[string]interface{}{
 303                                         "gateway_address": gwaddr}})
 304                 },
 305         }
 306         c.Assert(tungw.Start(), check.IsNil)
 307
 308         // We didn't supply an external hostname in the Address field,
 309         // so Start() should assign a local address.
 310         host, _, err := net.SplitHostPort(tungw.Address)
 311         c.Assert(err, check.IsNil)
 312         c.Check(host, check.Equals, "127.0.0.1")
 313
 314         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 315                 UUID: s.ctrUUID,
 316                 Attrs: map[string]interface{}{
 317                         "state": arvados.ContainerStateRunning,
 318                 }})
 319         c.Assert(err, check.IsNil)
 320
 321         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 322                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 323                 c.Assert(err, check.IsNil)
 324                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 325                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 326                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 327                         break
 328                 }
 329         }
 330
 331         c.Log("connecting to gateway through tunnel")
 332         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 333         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 334         if expectErrorMatch != "" {
 335                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 336                 return
 337         }
 338         c.Assert(err, check.IsNil)
 339         c.Assert(sshconn.Conn, check.NotNil)
 340         defer sshconn.Conn.Close()
 341
 342         done := make(chan struct{})
 343         go func() {
 344                 defer close(done)
 345
 346                 // Receive text banner
 347                 buf := make([]byte, 12)
 348                 _, err := io.ReadFull(sshconn.Conn, buf)
 349                 c.Check(err, check.IsNil)
 350                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 351
 352                 // Send text banner
 353                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 354                 c.Check(err, check.IsNil)
 355
 356                 // Receive binary
 357                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 358                 c.Check(err, check.IsNil)
 359
 360                 // If we can get this far into an SSH handshake...
 361                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 362         }()
 363         select {
 364         case <-done:
 365         case <-time.After(time.Second):
 366                 c.Fail()
 367         }
 368         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 369         c.Check(err, check.IsNil)
 370         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 371 }