lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "context"
   9         "crypto/hmac"
  10         "crypto/sha256"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http/httptest"
  16         "net/url"
  17         "strings"
  18         "time"
  19
  20         "git.arvados.org/arvados.git/lib/config"
  21         "git.arvados.org/arvados.git/lib/controller/router"
  22         "git.arvados.org/arvados.git/lib/controller/rpc"
  23         "git.arvados.org/arvados.git/lib/crunchrun"
  24         "git.arvados.org/arvados.git/sdk/go/arvados"
  25         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  26         "git.arvados.org/arvados.git/sdk/go/auth"
  27         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  28         "golang.org/x/crypto/ssh"
  29         check "gopkg.in/check.v1"
  30 )
  31
  32 var _ = check.Suite(&ContainerGatewaySuite{})
  33
  34 type ContainerGatewaySuite struct {
  35         cluster *arvados.Cluster
  36         localdb *Conn
  37         ctx     context.Context
  38         ctrUUID string
  39         gw      *crunchrun.Gateway
  40 }
  41
  42 func (s *ContainerGatewaySuite) TearDownSuite(c *check.C) {
  43         // Undo any changes/additions to the user database so they
  44         // don't affect subsequent tests.
  45         arvadostest.ResetEnv()
  46         c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
  47 }
  48
  49 func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
  50         cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
  51         c.Assert(err, check.IsNil)
  52         s.cluster, err = cfg.GetCluster("")
  53         c.Assert(err, check.IsNil)
  54         s.localdb = NewConn(s.cluster)
  55         s.ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
  56
  57         s.ctrUUID = arvadostest.QueuedContainerUUID
  58
  59         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
  60         fmt.Fprint(h, s.ctrUUID)
  61         authKey := fmt.Sprintf("%x", h.Sum(nil))
  62
  63         rtr := router.New(s.localdb, router.Config{})
  64         srv := httptest.NewUnstartedServer(rtr)
  65         srv.StartTLS()
  66         // the test setup doesn't use lib/service so
  67         // service.URLFromContext() returns nothing -- instead, this
  68         // is how we advertise our internal URL and enable
  69         // proxy-to-other-controller mode,
  70         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: srv.Listener.Addr().String()}
  71         ac := &arvados.Client{
  72                 APIHost:   srv.Listener.Addr().String(),
  73                 AuthToken: arvadostest.Dispatch1Token,
  74                 Insecure:  true,
  75         }
  76         s.gw = &crunchrun.Gateway{
  77                 ContainerUUID: s.ctrUUID,
  78                 AuthSecret:    authKey,
  79                 Address:       "localhost:0",
  80                 Log:           ctxlog.TestLogger(c),
  81                 Target:        crunchrun.GatewayTargetStub{},
  82                 ArvadosClient: ac,
  83         }
  84         c.Assert(s.gw.Start(), check.IsNil)
  85         rootctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
  86         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
  87                 UUID: s.ctrUUID,
  88                 Attrs: map[string]interface{}{
  89                         "state": arvados.ContainerStateLocked}})
  90         c.Assert(err, check.IsNil)
  91 }
  92
  93 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
  94         // clear any tunnel sessions started by previous test cases
  95         s.localdb.gwTunnelsLock.Lock()
  96         s.localdb.gwTunnels = nil
  97         s.localdb.gwTunnelsLock.Unlock()
  98
  99         rootctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
 100         _, err := s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 101                 UUID: s.ctrUUID,
 102                 Attrs: map[string]interface{}{
 103                         "state":           arvados.ContainerStateRunning,
 104                         "gateway_address": s.gw.Address}})
 105         c.Assert(err, check.IsNil)
 106
 107         s.cluster.Containers.ShellAccess.Admin = true
 108         s.cluster.Containers.ShellAccess.User = true
 109         _, err = arvadostest.DB(c, s.cluster).Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
 110         c.Check(err, check.IsNil)
 111 }
 112
 113 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 114         for _, trial := range []struct {
 115                 configAdmin bool
 116                 configUser  bool
 117                 sendToken   string
 118                 errorCode   int
 119         }{
 120                 {true, true, arvadostest.ActiveTokenV2, 0},
 121                 {true, false, arvadostest.ActiveTokenV2, 503},
 122                 {false, true, arvadostest.ActiveTokenV2, 0},
 123                 {false, false, arvadostest.ActiveTokenV2, 503},
 124                 {true, true, arvadostest.AdminToken, 0},
 125                 {true, false, arvadostest.AdminToken, 0},
 126                 {false, true, arvadostest.AdminToken, 403},
 127                 {false, false, arvadostest.AdminToken, 503},
 128         } {
 129                 c.Logf("trial %#v", trial)
 130                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 131                 s.cluster.Containers.ShellAccess.User = trial.configUser
 132                 ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{trial.sendToken}})
 133                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 134                 if trial.errorCode == 0 {
 135                         if !c.Check(err, check.IsNil) {
 136                                 continue
 137                         }
 138                         if !c.Check(sshconn.Conn, check.NotNil) {
 139                                 continue
 140                         }
 141                         sshconn.Conn.Close()
 142                 } else {
 143                         c.Check(err, check.NotNil)
 144                         err, ok := err.(interface{ HTTPStatus() int })
 145                         if c.Check(ok, check.Equals, true) {
 146                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 147                         }
 148                 }
 149         }
 150 }
 151
 152 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 153         // Set up servers on a few TCP ports
 154         var addrs []string
 155         for i := 0; i < 3; i++ {
 156                 ln, err := net.Listen("tcp", ":0")
 157                 c.Assert(err, check.IsNil)
 158                 defer ln.Close()
 159                 addrs = append(addrs, ln.Addr().String())
 160                 go func() {
 161                         for {
 162                                 conn, err := ln.Accept()
 163                                 if err != nil {
 164                                         return
 165                                 }
 166                                 var gotAddr string
 167                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 168                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 169                                 if gotAddr == ln.Addr().String() {
 170                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 171                                 }
 172                                 conn.Close()
 173                         }
 174                 }()
 175         }
 176
 177         c.Logf("connecting to %s", s.gw.Address)
 178         sshconn, err := s.localdb.ContainerSSH(s.ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 179         c.Assert(err, check.IsNil)
 180         c.Assert(sshconn.Conn, check.NotNil)
 181         defer sshconn.Conn.Close()
 182         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 183                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 184         })
 185         c.Assert(err, check.IsNil)
 186         client := ssh.NewClient(conn, chans, reqs)
 187         for _, expectAddr := range addrs {
 188                 _, port, err := net.SplitHostPort(expectAddr)
 189                 c.Assert(err, check.IsNil)
 190
 191                 c.Logf("trying foo:%s", port)
 192                 {
 193                         conn, err := client.Dial("tcp", "foo:"+port)
 194                         c.Assert(err, check.IsNil)
 195                         conn.SetDeadline(time.Now().Add(time.Second))
 196                         buf, err := ioutil.ReadAll(conn)
 197                         c.Check(err, check.IsNil)
 198                         c.Check(string(buf), check.Equals, "")
 199                 }
 200
 201                 c.Logf("trying localhost:%s", port)
 202                 {
 203                         conn, err := client.Dial("tcp", "localhost:"+port)
 204                         c.Assert(err, check.IsNil)
 205                         conn.SetDeadline(time.Now().Add(time.Second))
 206                         conn.Write([]byte(expectAddr + "\n"))
 207                         var gotAddr string
 208                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 209                         c.Check(gotAddr, check.Equals, expectAddr)
 210                 }
 211         }
 212 }
 213
 214 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 215         c.Logf("connecting to %s", s.gw.Address)
 216         sshconn, err := s.localdb.ContainerSSH(s.ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 217         c.Assert(err, check.IsNil)
 218         c.Assert(sshconn.Conn, check.NotNil)
 219         defer sshconn.Conn.Close()
 220
 221         done := make(chan struct{})
 222         go func() {
 223                 defer close(done)
 224
 225                 // Receive text banner
 226                 buf := make([]byte, 12)
 227                 _, err := io.ReadFull(sshconn.Conn, buf)
 228                 c.Check(err, check.IsNil)
 229                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 230
 231                 // Send text banner
 232                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 233                 c.Check(err, check.IsNil)
 234
 235                 // Receive binary
 236                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 237                 c.Check(err, check.IsNil)
 238
 239                 // If we can get this far into an SSH handshake...
 240                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 241         }()
 242         select {
 243         case <-done:
 244         case <-time.After(time.Second):
 245                 c.Fail()
 246         }
 247         ctr, err := s.localdb.ContainerGet(s.ctx, arvados.GetOptions{UUID: s.ctrUUID})
 248         c.Check(err, check.IsNil)
 249         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 250 }
 251
 252 func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
 253         c.Log("trying with no token")
 254         ctx := auth.NewContext(context.Background(), &auth.Credentials{})
 255         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 256         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 257
 258         c.Log("trying with anonymous token")
 259         ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AnonymousToken}})
 260         _, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 261         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 262 }
 263
 264 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 265         // no AuthSecret
 266         conn, err := s.localdb.ContainerGatewayTunnel(s.ctx, arvados.ContainerGatewayTunnelOptions{
 267                 UUID: s.ctrUUID,
 268         })
 269         c.Check(err, check.ErrorMatches, `authentication error`)
 270         c.Check(conn.Conn, check.IsNil)
 271
 272         // bogus AuthSecret
 273         conn, err = s.localdb.ContainerGatewayTunnel(s.ctx, arvados.ContainerGatewayTunnelOptions{
 274                 UUID:       s.ctrUUID,
 275                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 276         })
 277         c.Check(err, check.ErrorMatches, `authentication error`)
 278         c.Check(conn.Conn, check.IsNil)
 279
 280         // good AuthSecret
 281         conn, err = s.localdb.ContainerGatewayTunnel(s.ctx, arvados.ContainerGatewayTunnelOptions{
 282                 UUID:       s.ctrUUID,
 283                 AuthSecret: s.gw.AuthSecret,
 284         })
 285         c.Check(err, check.IsNil)
 286         c.Check(conn.Conn, check.NotNil)
 287 }
 288
 289 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 290         forceProxyForTest = true
 291         defer func() { forceProxyForTest = false }()
 292         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 293         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 294         s.testConnectThroughTunnel(c, "")
 295 }
 296
 297 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 298         forceProxyForTest = true
 299         defer func() { forceProxyForTest = false }()
 300         // forceInternalURLForTest shouldn't be used because it isn't
 301         // listed in s.cluster.Services.Controller.InternalURLs
 302         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 303 }
 304
 305 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 306         s.testConnectThroughTunnel(c, "")
 307 }
 308
 309 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 310         rootctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
 311         // Until the tunnel starts up, set gateway_address to a value
 312         // that can't work. We want to ensure the only way we can
 313         // reach the gateway is through the tunnel.
 314         gwaddr := "127.0.0.1:0"
 315         tungw := &crunchrun.Gateway{
 316                 ContainerUUID: s.ctrUUID,
 317                 AuthSecret:    s.gw.AuthSecret,
 318                 Log:           ctxlog.TestLogger(c),
 319                 Target:        crunchrun.GatewayTargetStub{},
 320                 ArvadosClient: s.gw.ArvadosClient,
 321                 UpdateTunnelURL: func(url string) {
 322                         c.Logf("UpdateTunnelURL(%q)", url)
 323                         gwaddr = "tunnel " + url
 324                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 325                                 UUID: s.ctrUUID,
 326                                 Attrs: map[string]interface{}{
 327                                         "gateway_address": gwaddr}})
 328                 },
 329         }
 330         c.Assert(tungw.Start(), check.IsNil)
 331
 332         // We didn't supply an external hostname in the Address field,
 333         // so Start() should assign a local address.
 334         host, _, err := net.SplitHostPort(tungw.Address)
 335         c.Assert(err, check.IsNil)
 336         c.Check(host, check.Equals, "127.0.0.1")
 337
 338         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 339                 UUID: s.ctrUUID,
 340                 Attrs: map[string]interface{}{
 341                         "state":           arvados.ContainerStateRunning,
 342                         "gateway_address": gwaddr}})
 343         c.Assert(err, check.IsNil)
 344
 345         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 346                 ctr, err := s.localdb.ContainerGet(s.ctx, arvados.GetOptions{UUID: s.ctrUUID})
 347                 c.Assert(err, check.IsNil)
 348                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 349                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 350                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 351                         break
 352                 }
 353         }
 354
 355         c.Log("connecting to gateway through tunnel")
 356         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 357         sshconn, err := arpc.ContainerSSH(s.ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 358         if expectErrorMatch != "" {
 359                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 360                 return
 361         }
 362         c.Assert(err, check.IsNil)
 363         c.Assert(sshconn.Conn, check.NotNil)
 364         defer sshconn.Conn.Close()
 365
 366         done := make(chan struct{})
 367         go func() {
 368                 defer close(done)
 369
 370                 // Receive text banner
 371                 buf := make([]byte, 12)
 372                 _, err := io.ReadFull(sshconn.Conn, buf)
 373                 c.Check(err, check.IsNil)
 374                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 375
 376                 // Send text banner
 377                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 378                 c.Check(err, check.IsNil)
 379
 380                 // Receive binary
 381                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 382                 c.Check(err, check.IsNil)
 383
 384                 // If we can get this far into an SSH handshake...
 385                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 386         }()
 387         select {
 388         case <-done:
 389         case <-time.After(time.Second):
 390                 c.Fail()
 391         }
 392         ctr, err := s.localdb.ContainerGet(s.ctx, arvados.GetOptions{UUID: s.ctrUUID})
 393         c.Check(err, check.IsNil)
 394         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 395 }