lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/hmac"
  11         "crypto/sha256"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "net"
  16         "net/http"
  17         "net/http/httptest"
  18         "net/url"
  19         "os"
  20         "os/exec"
  21         "path/filepath"
  22         "strings"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/controller/router"
  26         "git.arvados.org/arvados.git/lib/controller/rpc"
  27         "git.arvados.org/arvados.git/lib/crunchrun"
  28         "git.arvados.org/arvados.git/lib/ctrlctx"
  29         "git.arvados.org/arvados.git/sdk/go/arvados"
  30         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  31         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  32         "git.arvados.org/arvados.git/sdk/go/auth"
  33         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  34         "git.arvados.org/arvados.git/sdk/go/httpserver"
  35         "git.arvados.org/arvados.git/sdk/go/keepclient"
  36         "golang.org/x/crypto/ssh"
  37         check "gopkg.in/check.v1"
  38 )
  39
  40 var _ = check.Suite(&ContainerGatewaySuite{})
  41
  42 type ContainerGatewaySuite struct {
  43         localdbSuite
  44         reqUUID string
  45         ctrUUID string
  46         srv     *httptest.Server
  47         gw      *crunchrun.Gateway
  48 }
  49
  50 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
  51         s.localdbSuite.SetUpTest(c)
  52
  53         cr, err := s.localdb.ContainerRequestCreate(s.userctx, arvados.CreateOptions{
  54                 Attrs: map[string]interface{}{
  55                         "command":             []string{"echo", time.Now().Format(time.RFC3339Nano)},
  56                         "container_count_max": 1,
  57                         "container_image":     "arvados/apitestfixture:latest",
  58                         "cwd":                 "/tmp",
  59                         "environment":         map[string]string{},
  60                         "output_path":         "/out",
  61                         "priority":            1,
  62                         "state":               arvados.ContainerRequestStateCommitted,
  63                         "mounts": map[string]interface{}{
  64                                 "/out": map[string]interface{}{
  65                                         "kind":     "tmp",
  66                                         "capacity": 1000000,
  67                                 },
  68                         },
  69                         "runtime_constraints": map[string]interface{}{
  70                                 "vcpus": 1,
  71                                 "ram":   2,
  72                         }}})
  73         c.Assert(err, check.IsNil)
  74         s.reqUUID = cr.UUID
  75         s.ctrUUID = cr.ContainerUUID
  76
  77         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
  78         fmt.Fprint(h, s.ctrUUID)
  79         authKey := fmt.Sprintf("%x", h.Sum(nil))
  80
  81         rtr := router.New(s.localdb, router.Config{})
  82         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
  83         s.srv.StartTLS()
  84         // the test setup doesn't use lib/service so
  85         // service.URLFromContext() returns nothing -- instead, this
  86         // is how we advertise our internal URL and enable
  87         // proxy-to-other-controller mode,
  88         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
  89         ac := &arvados.Client{
  90                 APIHost:   s.srv.Listener.Addr().String(),
  91                 AuthToken: arvadostest.Dispatch1Token,
  92                 Insecure:  true,
  93         }
  94         s.gw = &crunchrun.Gateway{
  95                 ContainerUUID: s.ctrUUID,
  96                 AuthSecret:    authKey,
  97                 Address:       "localhost:0",
  98                 Log:           ctxlog.TestLogger(c),
  99                 Target:        crunchrun.GatewayTargetStub{},
 100                 ArvadosClient: ac,
 101         }
 102         c.Assert(s.gw.Start(), check.IsNil)
 103
 104         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 105         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 106                 UUID: s.ctrUUID,
 107                 Attrs: map[string]interface{}{
 108                         "state": arvados.ContainerStateLocked}})
 109         c.Assert(err, check.IsNil)
 110         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 111                 UUID: s.ctrUUID,
 112                 Attrs: map[string]interface{}{
 113                         "state":           arvados.ContainerStateRunning,
 114                         "gateway_address": s.gw.Address}})
 115         c.Assert(err, check.IsNil)
 116
 117         s.cluster.Containers.ShellAccess.Admin = true
 118         s.cluster.Containers.ShellAccess.User = true
 119         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
 120         c.Check(err, check.IsNil)
 121 }
 122
 123 func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
 124         s.srv.Close()
 125         s.localdbSuite.TearDownTest(c)
 126 }
 127
 128 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 129         for _, trial := range []struct {
 130                 configAdmin bool
 131                 configUser  bool
 132                 sendToken   string
 133                 errorCode   int
 134         }{
 135                 {true, true, arvadostest.ActiveTokenV2, 0},
 136                 {true, false, arvadostest.ActiveTokenV2, 503},
 137                 {false, true, arvadostest.ActiveTokenV2, 0},
 138                 {false, false, arvadostest.ActiveTokenV2, 503},
 139                 {true, true, arvadostest.AdminToken, 0},
 140                 {true, false, arvadostest.AdminToken, 0},
 141                 {false, true, arvadostest.AdminToken, 403},
 142                 {false, false, arvadostest.AdminToken, 503},
 143         } {
 144                 c.Logf("trial %#v", trial)
 145                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 146                 s.cluster.Containers.ShellAccess.User = trial.configUser
 147                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 148                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 149                 if trial.errorCode == 0 {
 150                         if !c.Check(err, check.IsNil) {
 151                                 continue
 152                         }
 153                         if !c.Check(sshconn.Conn, check.NotNil) {
 154                                 continue
 155                         }
 156                         sshconn.Conn.Close()
 157                 } else {
 158                         c.Check(err, check.NotNil)
 159                         err, ok := err.(interface{ HTTPStatus() int })
 160                         if c.Check(ok, check.Equals, true) {
 161                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 162                         }
 163                 }
 164         }
 165 }
 166
 167 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 168         // Set up servers on a few TCP ports
 169         var addrs []string
 170         for i := 0; i < 3; i++ {
 171                 ln, err := net.Listen("tcp", ":0")
 172                 c.Assert(err, check.IsNil)
 173                 defer ln.Close()
 174                 addrs = append(addrs, ln.Addr().String())
 175                 go func() {
 176                         for {
 177                                 conn, err := ln.Accept()
 178                                 if err != nil {
 179                                         return
 180                                 }
 181                                 var gotAddr string
 182                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 183                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 184                                 if gotAddr == ln.Addr().String() {
 185                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 186                                 }
 187                                 conn.Close()
 188                         }
 189                 }()
 190         }
 191
 192         c.Logf("connecting to %s", s.gw.Address)
 193         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 194         c.Assert(err, check.IsNil)
 195         c.Assert(sshconn.Conn, check.NotNil)
 196         defer sshconn.Conn.Close()
 197         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 198                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 199         })
 200         c.Assert(err, check.IsNil)
 201         client := ssh.NewClient(conn, chans, reqs)
 202         for _, expectAddr := range addrs {
 203                 _, port, err := net.SplitHostPort(expectAddr)
 204                 c.Assert(err, check.IsNil)
 205
 206                 c.Logf("trying foo:%s", port)
 207                 {
 208                         conn, err := client.Dial("tcp", "foo:"+port)
 209                         c.Assert(err, check.IsNil)
 210                         conn.SetDeadline(time.Now().Add(time.Second))
 211                         buf, err := ioutil.ReadAll(conn)
 212                         c.Check(err, check.IsNil)
 213                         c.Check(string(buf), check.Equals, "")
 214                 }
 215
 216                 c.Logf("trying localhost:%s", port)
 217                 {
 218                         conn, err := client.Dial("tcp", "localhost:"+port)
 219                         c.Assert(err, check.IsNil)
 220                         conn.SetDeadline(time.Now().Add(time.Second))
 221                         conn.Write([]byte(expectAddr + "\n"))
 222                         var gotAddr string
 223                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 224                         c.Check(gotAddr, check.Equals, expectAddr)
 225                 }
 226         }
 227 }
 228
 229 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
 230         files := map[string]string{
 231                 "stderr.txt":   "hello world\n",
 232                 "a/b/c/d.html": "<html></html>\n",
 233         }
 234         client := arvados.NewClientFromEnv()
 235         ac, err := arvadosclient.New(client)
 236         c.Assert(err, check.IsNil)
 237         kc, err := keepclient.MakeKeepClient(ac)
 238         c.Assert(err, check.IsNil)
 239         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 240         c.Assert(err, check.IsNil)
 241         for name, content := range files {
 242                 for i, ch := range name {
 243                         if ch == '/' {
 244                                 err := cfs.Mkdir("/"+name[:i], 0777)
 245                                 c.Assert(err, check.IsNil)
 246                         }
 247                 }
 248                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 249                 c.Assert(err, check.IsNil)
 250                 f.Write([]byte(content))
 251                 err = f.Close()
 252                 c.Assert(err, check.IsNil)
 253         }
 254         cfs.Sync()
 255         s.gw.LogCollection = cfs
 256 }
 257
 258 func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
 259         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 260         txt, err := s.gw.LogCollection.MarshalManifest(".")
 261         c.Assert(err, check.IsNil)
 262         coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 263                 Attrs: map[string]interface{}{
 264                         "manifest_text": txt,
 265                 }})
 266         c.Assert(err, check.IsNil)
 267         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 268                 UUID: s.ctrUUID,
 269                 Attrs: map[string]interface{}{
 270                         "state":     arvados.ContainerStateComplete,
 271                         "exit_code": 0,
 272                         "log":       coll.PortableDataHash,
 273                 }})
 274         c.Assert(err, check.IsNil)
 275         updatedReq, err := s.localdb.ContainerRequestGet(rootctx, arvados.GetOptions{UUID: s.reqUUID})
 276         c.Assert(err, check.IsNil)
 277         c.Logf("container request log UUID is %s", updatedReq.LogUUID)
 278         crLog, err := s.localdb.CollectionGet(rootctx, arvados.GetOptions{UUID: updatedReq.LogUUID, Select: []string{"manifest_text"}})
 279         c.Assert(err, check.IsNil)
 280         c.Logf("collection log manifest:\n%s", crLog.ManifestText)
 281         // Ensure localdb can't circumvent the keep-web proxy test by
 282         // getting content from the container gateway.
 283         s.gw.LogCollection = nil
 284 }
 285
 286 func (s *ContainerGatewaySuite) TestContainerRequestLogViaTunnel(c *check.C) {
 287         forceProxyForTest = true
 288         defer func() { forceProxyForTest = false }()
 289
 290         s.gw = s.setupGatewayWithTunnel(c)
 291         s.setupLogCollection(c)
 292
 293         for _, broken := range []bool{false, true} {
 294                 c.Logf("broken=%v", broken)
 295
 296                 if broken {
 297                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 298                 } else {
 299                         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 300                         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 301                 }
 302
 303                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID+"/stderr.txt", nil)
 304                 c.Assert(err, check.IsNil)
 305                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 306                 handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
 307                         UUID: s.reqUUID,
 308                         WebDAVOptions: arvados.WebDAVOptions{
 309                                 Method: "GET",
 310                                 Header: r.Header,
 311                                 Path:   "/" + s.ctrUUID + "/stderr.txt",
 312                         },
 313                 })
 314                 if broken {
 315                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 316                         continue
 317                 }
 318                 c.Check(err, check.IsNil)
 319                 c.Assert(handler, check.NotNil)
 320                 rec := httptest.NewRecorder()
 321                 handler.ServeHTTP(rec, r)
 322                 resp := rec.Result()
 323                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 324                 buf, err := ioutil.ReadAll(resp.Body)
 325                 c.Check(err, check.IsNil)
 326                 c.Check(string(buf), check.Equals, "hello world\n")
 327         }
 328 }
 329
 330 func (s *ContainerGatewaySuite) TestContainerRequestLogViaGateway(c *check.C) {
 331         s.setupLogCollection(c)
 332         s.testContainerRequestLog(c)
 333 }
 334
 335 func (s *ContainerGatewaySuite) TestContainerRequestLogViaKeepWeb(c *check.C) {
 336         s.setupLogCollection(c)
 337         s.saveLogAndCloseGateway(c)
 338         s.testContainerRequestLog(c)
 339 }
 340
 341 func (s *ContainerGatewaySuite) testContainerRequestLog(c *check.C) {
 342         for _, trial := range []struct {
 343                 method          string
 344                 path            string
 345                 header          http.Header
 346                 unauthenticated bool
 347                 expectStatus    int
 348                 expectBodyRe    string
 349                 expectHeader    http.Header
 350         }{
 351                 {
 352                         method:       "GET",
 353                         path:         s.ctrUUID + "/stderr.txt",
 354                         expectStatus: http.StatusOK,
 355                         expectBodyRe: "hello world\n",
 356                         expectHeader: http.Header{
 357                                 "Content-Type": {"text/plain; charset=utf-8"},
 358                         },
 359                 },
 360                 {
 361                         method: "GET",
 362                         path:   s.ctrUUID + "/stderr.txt",
 363                         header: http.Header{
 364                                 "Range": {"bytes=-6"},
 365                         },
 366                         expectStatus: http.StatusPartialContent,
 367                         expectBodyRe: "world\n",
 368                         expectHeader: http.Header{
 369                                 "Content-Type":  {"text/plain; charset=utf-8"},
 370                                 "Content-Range": {"bytes 6-11/12"},
 371                         },
 372                 },
 373                 {
 374                         method:       "OPTIONS",
 375                         path:         s.ctrUUID + "/stderr.txt",
 376                         expectStatus: http.StatusOK,
 377                         expectBodyRe: "",
 378                         expectHeader: http.Header{
 379                                 "Dav":   {"1, 2"},
 380                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 381                         },
 382                 },
 383                 {
 384                         method:          "OPTIONS",
 385                         path:            s.ctrUUID + "/stderr.txt",
 386                         unauthenticated: true,
 387                         header: http.Header{
 388                                 "Access-Control-Request-Method": {"POST"},
 389                         },
 390                         expectStatus: http.StatusOK,
 391                         expectBodyRe: "",
 392                         expectHeader: http.Header{
 393                                 "Access-Control-Allow-Headers": {"Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control"},
 394                                 "Access-Control-Allow-Methods": {"COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK"},
 395                                 "Access-Control-Allow-Origin":  {"*"},
 396                                 "Access-Control-Max-Age":       {"86400"},
 397                         },
 398                 },
 399                 {
 400                         method:       "PROPFIND",
 401                         path:         s.ctrUUID + "/",
 402                         expectStatus: http.StatusMultiStatus,
 403                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 404                         expectHeader: http.Header{
 405                                 "Content-Type": {"text/xml; charset=utf-8"},
 406                         },
 407                 },
 408                 {
 409                         method:       "PROPFIND",
 410                         path:         s.ctrUUID,
 411                         expectStatus: http.StatusMultiStatus,
 412                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 413                         expectHeader: http.Header{
 414                                 "Content-Type": {"text/xml; charset=utf-8"},
 415                         },
 416                 },
 417                 {
 418                         method:       "PROPFIND",
 419                         path:         s.ctrUUID + "/a/b/c/",
 420                         expectStatus: http.StatusMultiStatus,
 421                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*>\n?`,
 422                         expectHeader: http.Header{
 423                                 "Content-Type": {"text/xml; charset=utf-8"},
 424                         },
 425                 },
 426                 {
 427                         method:       "GET",
 428                         path:         s.ctrUUID + "/a/b/c/d.html",
 429                         expectStatus: http.StatusOK,
 430                         expectBodyRe: "<html></html>\n",
 431                         expectHeader: http.Header{
 432                                 "Content-Type": {"text/html; charset=utf-8"},
 433                         },
 434                 },
 435         } {
 436                 c.Logf("trial %#v", trial)
 437                 ctx := s.userctx
 438                 if trial.unauthenticated {
 439                         ctx = auth.NewContext(context.Background(), auth.CredentialsFromRequest(&http.Request{URL: &url.URL{}, Header: http.Header{}}))
 440                 }
 441                 r, err := http.NewRequestWithContext(ctx, trial.method, "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+trial.path, nil)
 442                 c.Assert(err, check.IsNil)
 443                 for k := range trial.header {
 444                         r.Header.Set(k, trial.header.Get(k))
 445                 }
 446                 handler, err := s.localdb.ContainerRequestLog(ctx, arvados.ContainerLogOptions{
 447                         UUID: s.reqUUID,
 448                         WebDAVOptions: arvados.WebDAVOptions{
 449                                 Method: trial.method,
 450                                 Header: r.Header,
 451                                 Path:   "/" + trial.path,
 452                         },
 453                 })
 454                 c.Assert(err, check.IsNil)
 455                 c.Assert(handler, check.NotNil)
 456                 rec := httptest.NewRecorder()
 457                 handler.ServeHTTP(rec, r)
 458                 resp := rec.Result()
 459                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
 460                 for k := range trial.expectHeader {
 461                         c.Check(resp.Header[k], check.DeepEquals, trial.expectHeader[k])
 462                 }
 463                 buf, err := ioutil.ReadAll(resp.Body)
 464                 c.Check(err, check.IsNil)
 465                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
 466         }
 467 }
 468
 469 func (s *ContainerGatewaySuite) TestContainerRequestLogViaCadaver(c *check.C) {
 470         s.setupLogCollection(c)
 471
 472         out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "ls")
 473         c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
 474         c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
 475
 476         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
 477         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 478
 479         s.saveLogAndCloseGateway(c)
 480
 481         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
 482         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 483 }
 484
 485 func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
 486         // Replace s.srv with an HTTP server, otherwise cadaver will
 487         // just fail on TLS cert verification.
 488         s.srv.Close()
 489         rtr := router.New(s.localdb, router.Config{})
 490         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
 491         s.srv.Start()
 492
 493         tempdir, err := ioutil.TempDir("", "localdb-test-")
 494         c.Assert(err, check.IsNil)
 495         defer os.RemoveAll(tempdir)
 496
 497         cmd := exec.Command("cadaver", s.srv.URL+path)
 498         if password != "" {
 499                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 500                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 501                 c.Assert(err, check.IsNil)
 502                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 503                 c.Assert(err, check.IsNil)
 504                 c.Assert(f.Close(), check.IsNil)
 505         }
 506         cmd.Stdin = bytes.NewBufferString(stdin)
 507         cmd.Dir = tempdir
 508         stdout, err := cmd.StdoutPipe()
 509         c.Assert(err, check.Equals, nil)
 510         cmd.Stderr = cmd.Stdout
 511         c.Logf("cmd: %v", cmd.Args)
 512         go cmd.Start()
 513
 514         var buf bytes.Buffer
 515         _, err = io.Copy(&buf, stdout)
 516         c.Check(err, check.Equals, nil)
 517         err = cmd.Wait()
 518         c.Check(err, check.Equals, nil)
 519         return buf.String()
 520 }
 521
 522 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 523         c.Logf("connecting to %s", s.gw.Address)
 524         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 525         c.Assert(err, check.IsNil)
 526         c.Assert(sshconn.Conn, check.NotNil)
 527         defer sshconn.Conn.Close()
 528
 529         done := make(chan struct{})
 530         go func() {
 531                 defer close(done)
 532
 533                 // Receive text banner
 534                 buf := make([]byte, 12)
 535                 _, err := io.ReadFull(sshconn.Conn, buf)
 536                 c.Check(err, check.IsNil)
 537                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 538
 539                 // Send text banner
 540                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 541                 c.Check(err, check.IsNil)
 542
 543                 // Receive binary
 544                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 545                 c.Check(err, check.IsNil)
 546
 547                 // If we can get this far into an SSH handshake...
 548                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 549         }()
 550         select {
 551         case <-done:
 552         case <-time.After(time.Second):
 553                 c.Fail()
 554         }
 555         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 556         c.Check(err, check.IsNil)
 557         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 558 }
 559
 560 func (s *ContainerGatewaySuite) TestConnectFail(c *check.C) {
 561         c.Log("trying with no token")
 562         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
 563         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 564         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 565
 566         c.Log("trying with anonymous token")
 567         ctx = ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
 568         _, err = s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 569         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 570 }
 571
 572 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 573         // no AuthSecret
 574         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 575                 UUID: s.ctrUUID,
 576         })
 577         c.Check(err, check.ErrorMatches, `authentication error`)
 578         c.Check(conn.Conn, check.IsNil)
 579
 580         // bogus AuthSecret
 581         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 582                 UUID:       s.ctrUUID,
 583                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 584         })
 585         c.Check(err, check.ErrorMatches, `authentication error`)
 586         c.Check(conn.Conn, check.IsNil)
 587
 588         // good AuthSecret
 589         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 590                 UUID:       s.ctrUUID,
 591                 AuthSecret: s.gw.AuthSecret,
 592         })
 593         c.Check(err, check.IsNil)
 594         c.Check(conn.Conn, check.NotNil)
 595 }
 596
 597 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 598         forceProxyForTest = true
 599         defer func() { forceProxyForTest = false }()
 600         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 601         defer delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 602         s.testConnectThroughTunnel(c, "")
 603 }
 604
 605 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 606         forceProxyForTest = true
 607         defer func() { forceProxyForTest = false }()
 608         // forceInternalURLForTest will not be usable because it isn't
 609         // listed in s.cluster.Services.Controller.InternalURLs
 610         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 611 }
 612
 613 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 614         s.testConnectThroughTunnel(c, "")
 615 }
 616
 617 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
 618         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 619         // Until the tunnel starts up, set gateway_address to a value
 620         // that can't work. We want to ensure the only way we can
 621         // reach the gateway is through the tunnel.
 622         tungw := &crunchrun.Gateway{
 623                 ContainerUUID: s.ctrUUID,
 624                 AuthSecret:    s.gw.AuthSecret,
 625                 Log:           ctxlog.TestLogger(c),
 626                 Target:        crunchrun.GatewayTargetStub{},
 627                 ArvadosClient: s.gw.ArvadosClient,
 628                 UpdateTunnelURL: func(url string) {
 629                         c.Logf("UpdateTunnelURL(%q)", url)
 630                         gwaddr := "tunnel " + url
 631                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 632                                 UUID: s.ctrUUID,
 633                                 Attrs: map[string]interface{}{
 634                                         "gateway_address": gwaddr}})
 635                 },
 636         }
 637         c.Assert(tungw.Start(), check.IsNil)
 638
 639         // We didn't supply an external hostname in the Address field,
 640         // so Start() should assign a local address.
 641         host, _, err := net.SplitHostPort(tungw.Address)
 642         c.Assert(err, check.IsNil)
 643         c.Check(host, check.Equals, "127.0.0.1")
 644
 645         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 646                 UUID: s.ctrUUID,
 647                 Attrs: map[string]interface{}{
 648                         "state": arvados.ContainerStateRunning,
 649                 }})
 650         c.Assert(err, check.IsNil)
 651
 652         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 653                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 654                 c.Assert(err, check.IsNil)
 655                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 656                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 657                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 658                         break
 659                 }
 660         }
 661         return tungw
 662 }
 663
 664 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 665         s.setupGatewayWithTunnel(c)
 666         c.Log("connecting to gateway through tunnel")
 667         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 668         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 669         if expectErrorMatch != "" {
 670                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 671                 return
 672         }
 673         c.Assert(err, check.IsNil)
 674         c.Assert(sshconn.Conn, check.NotNil)
 675         defer sshconn.Conn.Close()
 676
 677         done := make(chan struct{})
 678         go func() {
 679                 defer close(done)
 680
 681                 // Receive text banner
 682                 buf := make([]byte, 12)
 683                 _, err := io.ReadFull(sshconn.Conn, buf)
 684                 c.Check(err, check.IsNil)
 685                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 686
 687                 // Send text banner
 688                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 689                 c.Check(err, check.IsNil)
 690
 691                 // Receive binary
 692                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 693                 c.Check(err, check.IsNil)
 694
 695                 // If we can get this far into an SSH handshake...
 696                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 697         }()
 698         select {
 699         case <-done:
 700         case <-time.After(time.Second):
 701                 c.Fail()
 702         }
 703         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 704         c.Check(err, check.IsNil)
 705         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 706 }