lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/hmac"
  11         "crypto/sha256"
  12         "errors"
  13         "fmt"
  14         "io"
  15         "io/ioutil"
  16         "net"
  17         "net/http"
  18         "net/http/httptest"
  19         "net/url"
  20         "os"
  21         "os/exec"
  22         "path/filepath"
  23         "strconv"
  24         "strings"
  25         "sync"
  26         "time"
  27
  28         "git.arvados.org/arvados.git/lib/controller/router"
  29         "git.arvados.org/arvados.git/lib/controller/rpc"
  30         "git.arvados.org/arvados.git/lib/crunchrun"
  31         "git.arvados.org/arvados.git/lib/ctrlctx"
  32         "git.arvados.org/arvados.git/sdk/go/arvados"
  33         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  34         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  35         "git.arvados.org/arvados.git/sdk/go/auth"
  36         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  37         "git.arvados.org/arvados.git/sdk/go/httpserver"
  38         "git.arvados.org/arvados.git/sdk/go/keepclient"
  39         "golang.org/x/crypto/ssh"
  40         check "gopkg.in/check.v1"
  41 )
  42
  43 var _ = check.Suite(&ContainerGatewaySuite{})
  44
  45 type ContainerGatewaySuite struct {
  46         localdbSuite
  47         reqCreateOptions arvados.CreateOptions
  48         reqUUID          string
  49         ctrUUID          string
  50         srv              *httptest.Server
  51         gw               *crunchrun.Gateway
  52 }
  53
  54 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
  55         s.localdbSuite.SetUpTest(c)
  56
  57         s.reqCreateOptions = arvados.CreateOptions{
  58                 Attrs: map[string]interface{}{
  59                         "command":             []string{"echo", time.Now().Format(time.RFC3339Nano)},
  60                         "container_count_max": 1,
  61                         "container_image":     "arvados/apitestfixture:latest",
  62                         "cwd":                 "/tmp",
  63                         "environment":         map[string]string{},
  64                         "output_path":         "/out",
  65                         "priority":            1,
  66                         "state":               arvados.ContainerRequestStateCommitted,
  67                         "mounts": map[string]interface{}{
  68                                 "/out": map[string]interface{}{
  69                                         "kind":     "tmp",
  70                                         "capacity": 1000000,
  71                                 },
  72                         },
  73                         "runtime_constraints": map[string]interface{}{
  74                                 "vcpus": 1,
  75                                 "ram":   2,
  76                         }}}
  77         cr, err := s.localdb.ContainerRequestCreate(s.userctx, s.reqCreateOptions)
  78         c.Assert(err, check.IsNil)
  79         s.reqUUID = cr.UUID
  80         s.ctrUUID = cr.ContainerUUID
  81
  82         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
  83         fmt.Fprint(h, s.ctrUUID)
  84         authKey := fmt.Sprintf("%x", h.Sum(nil))
  85
  86         rtr := router.New(s.localdb, router.Config{})
  87         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
  88         s.srv.StartTLS()
  89         // the test setup doesn't use lib/service so
  90         // service.URLFromContext() returns nothing -- instead, this
  91         // is how we advertise our internal URL and enable
  92         // proxy-to-other-controller mode,
  93         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
  94         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
  95         ac := &arvados.Client{
  96                 APIHost:   s.srv.Listener.Addr().String(),
  97                 AuthToken: arvadostest.SystemRootToken,
  98                 Insecure:  true,
  99         }
 100         s.gw = &crunchrun.Gateway{
 101                 ContainerUUID: s.ctrUUID,
 102                 AuthSecret:    authKey,
 103                 Address:       "localhost:0",
 104                 Log:           ctxlog.TestLogger(c),
 105                 Target:        crunchrun.GatewayTargetStub{},
 106                 ArvadosClient: ac,
 107         }
 108         c.Assert(s.gw.Start(), check.IsNil)
 109
 110         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 111         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 112                 UUID: s.ctrUUID,
 113                 Attrs: map[string]interface{}{
 114                         "state": arvados.ContainerStateLocked}})
 115         c.Assert(err, check.IsNil)
 116         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 117                 UUID: s.ctrUUID,
 118                 Attrs: map[string]interface{}{
 119                         "state":           arvados.ContainerStateRunning,
 120                         "gateway_address": s.gw.Address}})
 121         c.Assert(err, check.IsNil)
 122
 123         s.cluster.Containers.ShellAccess.Admin = true
 124         s.cluster.Containers.ShellAccess.User = true
 125         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
 126         c.Check(err, check.IsNil)
 127 }
 128
 129 func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
 130         forceProxyForTest = false
 131         s.srv.Close()
 132         s.localdbSuite.TearDownTest(c)
 133 }
 134
 135 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 136         for _, trial := range []struct {
 137                 configAdmin bool
 138                 configUser  bool
 139                 sendToken   string
 140                 errorCode   int
 141         }{
 142                 {true, true, arvadostest.ActiveTokenV2, 0},
 143                 {true, false, arvadostest.ActiveTokenV2, 503},
 144                 {false, true, arvadostest.ActiveTokenV2, 0},
 145                 {false, false, arvadostest.ActiveTokenV2, 503},
 146                 {true, true, arvadostest.AdminToken, 0},
 147                 {true, false, arvadostest.AdminToken, 0},
 148                 {false, true, arvadostest.AdminToken, 403},
 149                 {false, false, arvadostest.AdminToken, 503},
 150         } {
 151                 c.Logf("trial %#v", trial)
 152                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 153                 s.cluster.Containers.ShellAccess.User = trial.configUser
 154                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 155                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 156                 if trial.errorCode == 0 {
 157                         if !c.Check(err, check.IsNil) {
 158                                 continue
 159                         }
 160                         if !c.Check(sshconn.Conn, check.NotNil) {
 161                                 continue
 162                         }
 163                         sshconn.Conn.Close()
 164                 } else {
 165                         c.Check(err, check.NotNil)
 166                         err, ok := err.(interface{ HTTPStatus() int })
 167                         if c.Check(ok, check.Equals, true) {
 168                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 169                         }
 170                 }
 171         }
 172 }
 173
 174 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 175         // Set up servers on a few TCP ports
 176         var addrs []string
 177         for i := 0; i < 3; i++ {
 178                 ln, err := net.Listen("tcp", ":0")
 179                 c.Assert(err, check.IsNil)
 180                 defer ln.Close()
 181                 addrs = append(addrs, ln.Addr().String())
 182                 go func() {
 183                         for {
 184                                 conn, err := ln.Accept()
 185                                 if err != nil {
 186                                         return
 187                                 }
 188                                 var gotAddr string
 189                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 190                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 191                                 if gotAddr == ln.Addr().String() {
 192                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 193                                 }
 194                                 conn.Close()
 195                         }
 196                 }()
 197         }
 198
 199         c.Logf("connecting to %s", s.gw.Address)
 200         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 201         c.Assert(err, check.IsNil)
 202         c.Assert(sshconn.Conn, check.NotNil)
 203         defer sshconn.Conn.Close()
 204         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 205                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 206         })
 207         c.Assert(err, check.IsNil)
 208         client := ssh.NewClient(conn, chans, reqs)
 209         for _, expectAddr := range addrs {
 210                 _, port, err := net.SplitHostPort(expectAddr)
 211                 c.Assert(err, check.IsNil)
 212
 213                 c.Logf("trying foo:%s", port)
 214                 {
 215                         conn, err := client.Dial("tcp", "foo:"+port)
 216                         c.Assert(err, check.IsNil)
 217                         conn.SetDeadline(time.Now().Add(time.Second))
 218                         buf, err := ioutil.ReadAll(conn)
 219                         c.Check(err, check.IsNil)
 220                         c.Check(string(buf), check.Equals, "")
 221                 }
 222
 223                 c.Logf("trying localhost:%s", port)
 224                 {
 225                         conn, err := client.Dial("tcp", "localhost:"+port)
 226                         c.Assert(err, check.IsNil)
 227                         conn.SetDeadline(time.Now().Add(time.Second))
 228                         conn.Write([]byte(expectAddr + "\n"))
 229                         var gotAddr string
 230                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 231                         c.Check(gotAddr, check.Equals, expectAddr)
 232                 }
 233         }
 234 }
 235
 236 // Connect to crunch-run container gateway directly.
 237 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Direct(c *check.C) {
 238         s.testContainerHTTPProxy(c)
 239 }
 240
 241 // Connect through a tunnel terminated at this controller process.
 242 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Tunnel(c *check.C) {
 243         s.gw = s.setupGatewayWithTunnel(c)
 244         s.testContainerHTTPProxy(c)
 245 }
 246
 247 // Connect through a tunnel terminated at a different controller
 248 // process.
 249 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ProxyTunnel(c *check.C) {
 250         forceProxyForTest = true
 251         s.gw = s.setupGatewayWithTunnel(c)
 252         s.testContainerHTTPProxy(c)
 253 }
 254
 255 func (s *ContainerGatewaySuite) testContainerHTTPProxy(c *check.C) {
 256         var servers []*httpserver.Server
 257         for i := 0; i < 10; i++ {
 258                 srv := &httpserver.Server{
 259                         Addr: ":0",
 260                         Server: http.Server{
 261                                 Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 262                                         w.WriteHeader(http.StatusOK)
 263                                         body := fmt.Sprintf("handled %s %s with Host %s", r.Method, r.URL.String(), r.Host)
 264                                         c.Logf("%s", body)
 265                                         w.Write([]byte(body))
 266                                 }),
 267                         },
 268                 }
 269                 srv.Start()
 270                 defer srv.Close()
 271                 servers = append(servers, srv)
 272         }
 273
 274         testMethods := []string{"GET", "POST", "PATCH", "OPTIONS", "DELETE"}
 275
 276         var wg sync.WaitGroup
 277         for idx, srv := range servers {
 278                 idx, srv := idx, srv
 279                 wg.Add(1)
 280                 go func() {
 281                         defer wg.Done()
 282                         c.Logf("sending request to %s via %s", srv.Addr, s.gw.Address)
 283                         method := testMethods[idx%len(testMethods)]
 284                         _, port, err := net.SplitHostPort(srv.Addr)
 285                         c.Assert(err, check.IsNil)
 286                         vhost := s.ctrUUID + "-" + port + ".containers.example.com"
 287                         req, err := http.NewRequest(method, "https://"+vhost+"/via-"+s.gw.Address, nil)
 288                         c.Assert(err, check.IsNil)
 289                         // Token is already passed to
 290                         // ContainerHTTPProxy() call in s.userctx, but
 291                         // we also need to add an auth cookie to the
 292                         // http request: if the request gets passed
 293                         // through http (see forceProxyForTest), the
 294                         // target router will start with a fresh
 295                         // context and load tokens from the request.
 296                         req.AddCookie(&http.Cookie{
 297                                 Name:  "arvados_api_token",
 298                                 Value: auth.EncodeTokenCookie([]byte(arvadostest.ActiveTokenV2)),
 299                         })
 300                         portnum, err := strconv.Atoi(port)
 301                         c.Assert(err, check.IsNil)
 302                         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 303                                 UUID:    s.ctrUUID,
 304                                 Port:    portnum,
 305                                 Request: req,
 306                         })
 307                         c.Assert(err, check.IsNil)
 308                         rw := httptest.NewRecorder()
 309                         handler.ServeHTTP(rw, req)
 310                         resp := rw.Result()
 311                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 312                         body, err := io.ReadAll(resp.Body)
 313                         c.Assert(err, check.IsNil)
 314                         c.Check(string(body), check.Matches, `handled `+method+` /via-.* with Host \Q`+vhost+`\E`)
 315                 }()
 316         }
 317         wg.Wait()
 318 }
 319
 320 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_NoToken(c *check.C) {
 321         s.testContainerHTTPProxyError(c, "", http.StatusUnauthorized)
 322 }
 323
 324 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_InvalidToken(c *check.C) {
 325         s.testContainerHTTPProxyError(c, arvadostest.ActiveTokenV2+"bogus", http.StatusUnauthorized)
 326 }
 327
 328 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_AnonymousToken(c *check.C) {
 329         s.testContainerHTTPProxyError(c, arvadostest.AnonymousToken, http.StatusNotFound)
 330 }
 331
 332 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_CRsDifferentUsers(c *check.C) {
 333         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 334         cr, err := s.localdb.ContainerRequestCreate(rootctx, s.reqCreateOptions)
 335         c.Assert(err, check.IsNil)
 336         c.Assert(cr.ContainerUUID, check.Equals, s.ctrUUID)
 337         s.testContainerHTTPProxyError(c, arvadostest.ActiveTokenV2, http.StatusForbidden)
 338 }
 339
 340 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_ContainerNotReadable(c *check.C) {
 341         s.testContainerHTTPProxyError(c, arvadostest.SpectatorToken, http.StatusNotFound)
 342 }
 343
 344 func (s *ContainerGatewaySuite) testContainerHTTPProxyError(c *check.C, token string, expectCode int) {
 345         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, token)
 346         vhost := s.ctrUUID + "-12345.containers.example.com"
 347         req, err := http.NewRequest("GET", "https://"+vhost+"/via-"+s.gw.Address, nil)
 348         c.Assert(err, check.IsNil)
 349         _, err = s.localdb.ContainerHTTPProxy(ctx, arvados.ContainerHTTPProxyOptions{
 350                 UUID:    s.ctrUUID,
 351                 Port:    12345,
 352                 Request: req,
 353         })
 354         se := httpserver.HTTPStatusError(nil)
 355         c.Assert(errors.As(err, &se), check.Equals, true)
 356         c.Check(se.HTTPStatus(), check.Equals, expectCode)
 357 }
 358
 359 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_CookieAuth(c *check.C) {
 360         s.testContainerHTTPProxyUsingCurl(c, arvadostest.ActiveTokenV2, "GET", "/foobar")
 361 }
 362
 363 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth(c *check.C) {
 364         s.testContainerHTTPProxyUsingCurl(c, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 365 }
 366
 367 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_Tunnel(c *check.C) {
 368         s.gw = s.setupGatewayWithTunnel(c)
 369         s.testContainerHTTPProxyUsingCurl(c, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 370 }
 371
 372 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_ProxyTunnel(c *check.C) {
 373         forceProxyForTest = true
 374         s.gw = s.setupGatewayWithTunnel(c)
 375         s.testContainerHTTPProxyUsingCurl(c, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 376 }
 377
 378 // Check other query parameters are preserved in the
 379 // redirect-with-cookie.
 380 //
 381 // Note the original request has "?baz&baz&..." and this changes to
 382 // "?baz=&baz=&..." in the redirect location.  We trust the target
 383 // service won't be sensitive to this difference.
 384 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_PreserveQuery(c *check.C) {
 385         body := s.testContainerHTTPProxyUsingCurl(c, "", "GET", "/foobar?baz&baz&arvados_api_token="+arvadostest.ActiveTokenV2+"&waz=quux")
 386         c.Check(body, check.Matches, `handled GET /foobar\?baz=&baz=&waz=quux with Host `+s.ctrUUID+`.*`)
 387 }
 388
 389 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Patch(c *check.C) {
 390         body := s.testContainerHTTPProxyUsingCurl(c, arvadostest.ActiveTokenV2, "PATCH", "/foobar")
 391         c.Check(body, check.Matches, `handled PATCH /foobar with Host `+s.ctrUUID+`.*`)
 392 }
 393
 394 func (s *ContainerGatewaySuite) testContainerHTTPProxyUsingCurl(c *check.C, cookietoken, method, path string) string {
 395         srv := &httpserver.Server{
 396                 Addr: ":0",
 397                 Server: http.Server{
 398                         Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 399                                 w.WriteHeader(http.StatusOK)
 400                                 body := fmt.Sprintf("handled %s %s with Host %s", r.Method, r.URL.String(), r.Host)
 401                                 c.Logf("%s", body)
 402                                 w.Write([]byte(body))
 403                         }),
 404                 },
 405         }
 406         srv.Start()
 407         defer srv.Close()
 408         _, srvPort, err := net.SplitHostPort(srv.Addr)
 409         c.Assert(err, check.IsNil)
 410
 411         vhost, err := url.Parse(s.srv.URL)
 412         c.Assert(err, check.IsNil)
 413         controllerHost := vhost.Host
 414         vhost.Host = s.ctrUUID + "-" + srvPort + ".containers.example.com"
 415         target, err := vhost.Parse(path)
 416         c.Assert(err, check.IsNil)
 417
 418         tempdir, err := ioutil.TempDir("", "localdb-test-")
 419         c.Assert(err, check.IsNil)
 420         defer os.RemoveAll(tempdir)
 421
 422         cmd := exec.Command("curl")
 423         if cookietoken != "" {
 424                 cmd.Args = append(cmd.Args, "--cookie", "arvados_api_token="+string(auth.EncodeTokenCookie([]byte(cookietoken))))
 425         } else {
 426                 cmd.Args = append(cmd.Args, "--cookie-jar", filepath.Join(tempdir, "cookie.jar"))
 427         }
 428         if method != "GET" {
 429                 cmd.Args = append(cmd.Args, "--request", method)
 430         }
 431         cmd.Args = append(cmd.Args, "--silent", "--insecure", "--location", "--connect-to", vhost.Hostname()+":443:"+controllerHost, target.String())
 432         cmd.Dir = tempdir
 433         stdout, err := cmd.StdoutPipe()
 434         c.Assert(err, check.Equals, nil)
 435         cmd.Stderr = cmd.Stdout
 436         c.Logf("cmd: %v", cmd.Args)
 437         go cmd.Start()
 438
 439         var buf bytes.Buffer
 440         _, err = io.Copy(&buf, stdout)
 441         c.Check(err, check.Equals, nil)
 442         err = cmd.Wait()
 443         c.Check(err, check.Equals, nil)
 444         c.Check(buf.String(), check.Matches, `handled `+method+` /.*`)
 445         return buf.String()
 446 }
 447
 448 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
 449         files := map[string]string{
 450                 "stderr.txt":   "hello world\n",
 451                 "a/b/c/d.html": "<html></html>\n",
 452         }
 453         client := arvados.NewClientFromEnv()
 454         ac, err := arvadosclient.New(client)
 455         c.Assert(err, check.IsNil)
 456         kc, err := keepclient.MakeKeepClient(ac)
 457         c.Assert(err, check.IsNil)
 458         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 459         c.Assert(err, check.IsNil)
 460         for name, content := range files {
 461                 for i, ch := range name {
 462                         if ch == '/' {
 463                                 err := cfs.Mkdir("/"+name[:i], 0777)
 464                                 c.Assert(err, check.IsNil)
 465                         }
 466                 }
 467                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 468                 c.Assert(err, check.IsNil)
 469                 f.Write([]byte(content))
 470                 err = f.Close()
 471                 c.Assert(err, check.IsNil)
 472         }
 473         cfs.Sync()
 474         s.gw.LogCollection = cfs
 475 }
 476
 477 func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
 478         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 479         txt, err := s.gw.LogCollection.MarshalManifest(".")
 480         c.Assert(err, check.IsNil)
 481         coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 482                 Attrs: map[string]interface{}{
 483                         "manifest_text": txt,
 484                 }})
 485         c.Assert(err, check.IsNil)
 486         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 487                 UUID: s.ctrUUID,
 488                 Attrs: map[string]interface{}{
 489                         "state":     arvados.ContainerStateComplete,
 490                         "exit_code": 0,
 491                         "log":       coll.PortableDataHash,
 492                 }})
 493         c.Assert(err, check.IsNil)
 494         updatedReq, err := s.localdb.ContainerRequestGet(rootctx, arvados.GetOptions{UUID: s.reqUUID})
 495         c.Assert(err, check.IsNil)
 496         c.Logf("container request log UUID is %s", updatedReq.LogUUID)
 497         crLog, err := s.localdb.CollectionGet(rootctx, arvados.GetOptions{UUID: updatedReq.LogUUID, Select: []string{"manifest_text"}})
 498         c.Assert(err, check.IsNil)
 499         c.Logf("collection log manifest:\n%s", crLog.ManifestText)
 500         // Ensure localdb can't circumvent the keep-web proxy test by
 501         // getting content from the container gateway.
 502         s.gw.LogCollection = nil
 503 }
 504
 505 func (s *ContainerGatewaySuite) TestContainerRequestLogViaTunnel(c *check.C) {
 506         forceProxyForTest = true
 507         s.gw = s.setupGatewayWithTunnel(c)
 508         s.setupLogCollection(c)
 509
 510         for _, broken := range []bool{false, true} {
 511                 c.Logf("broken=%v", broken)
 512
 513                 if broken {
 514                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 515                 }
 516
 517                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID+"/stderr.txt", nil)
 518                 c.Assert(err, check.IsNil)
 519                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 520                 handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
 521                         UUID: s.reqUUID,
 522                         WebDAVOptions: arvados.WebDAVOptions{
 523                                 Method: "GET",
 524                                 Header: r.Header,
 525                                 Path:   "/" + s.ctrUUID + "/stderr.txt",
 526                         },
 527                 })
 528                 if broken {
 529                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 530                         continue
 531                 }
 532                 c.Check(err, check.IsNil)
 533                 c.Assert(handler, check.NotNil)
 534                 rec := httptest.NewRecorder()
 535                 handler.ServeHTTP(rec, r)
 536                 resp := rec.Result()
 537                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 538                 buf, err := ioutil.ReadAll(resp.Body)
 539                 c.Check(err, check.IsNil)
 540                 c.Check(string(buf), check.Equals, "hello world\n")
 541         }
 542 }
 543
 544 func (s *ContainerGatewaySuite) TestContainerRequestLogViaGateway(c *check.C) {
 545         s.setupLogCollection(c)
 546         s.testContainerRequestLog(c)
 547 }
 548
 549 func (s *ContainerGatewaySuite) TestContainerRequestLogViaKeepWeb(c *check.C) {
 550         s.setupLogCollection(c)
 551         s.saveLogAndCloseGateway(c)
 552         s.testContainerRequestLog(c)
 553 }
 554
 555 func (s *ContainerGatewaySuite) testContainerRequestLog(c *check.C) {
 556         for _, trial := range []struct {
 557                 method          string
 558                 path            string
 559                 header          http.Header
 560                 unauthenticated bool
 561                 expectStatus    int
 562                 expectBodyRe    string
 563                 expectHeader    http.Header
 564         }{
 565                 {
 566                         method:       "GET",
 567                         path:         s.ctrUUID + "/stderr.txt",
 568                         expectStatus: http.StatusOK,
 569                         expectBodyRe: "hello world\n",
 570                         expectHeader: http.Header{
 571                                 "Content-Type": {"text/plain; charset=utf-8"},
 572                         },
 573                 },
 574                 {
 575                         method: "GET",
 576                         path:   s.ctrUUID + "/stderr.txt",
 577                         header: http.Header{
 578                                 "Range": {"bytes=-6"},
 579                         },
 580                         expectStatus: http.StatusPartialContent,
 581                         expectBodyRe: "world\n",
 582                         expectHeader: http.Header{
 583                                 "Content-Type":  {"text/plain; charset=utf-8"},
 584                                 "Content-Range": {"bytes 6-11/12"},
 585                         },
 586                 },
 587                 {
 588                         method:       "OPTIONS",
 589                         path:         s.ctrUUID + "/stderr.txt",
 590                         expectStatus: http.StatusOK,
 591                         expectBodyRe: "",
 592                         expectHeader: http.Header{
 593                                 "Dav":   {"1, 2"},
 594                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 595                         },
 596                 },
 597                 {
 598                         method:          "OPTIONS",
 599                         path:            s.ctrUUID + "/stderr.txt",
 600                         unauthenticated: true,
 601                         header: http.Header{
 602                                 "Access-Control-Request-Method": {"POST"},
 603                         },
 604                         expectStatus: http.StatusOK,
 605                         expectBodyRe: "",
 606                         expectHeader: http.Header{
 607                                 "Access-Control-Allow-Headers": {"Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control"},
 608                                 "Access-Control-Allow-Methods": {"COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK"},
 609                                 "Access-Control-Allow-Origin":  {"*"},
 610                                 "Access-Control-Max-Age":       {"86400"},
 611                         },
 612                 },
 613                 {
 614                         method:       "PROPFIND",
 615                         path:         s.ctrUUID + "/",
 616                         expectStatus: http.StatusMultiStatus,
 617                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 618                         expectHeader: http.Header{
 619                                 "Content-Type": {"text/xml; charset=utf-8"},
 620                         },
 621                 },
 622                 {
 623                         method:       "PROPFIND",
 624                         path:         s.ctrUUID,
 625                         expectStatus: http.StatusMultiStatus,
 626                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 627                         expectHeader: http.Header{
 628                                 "Content-Type": {"text/xml; charset=utf-8"},
 629                         },
 630                 },
 631                 {
 632                         method:       "PROPFIND",
 633                         path:         s.ctrUUID + "/a/b/c/",
 634                         expectStatus: http.StatusMultiStatus,
 635                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*>\n?`,
 636                         expectHeader: http.Header{
 637                                 "Content-Type": {"text/xml; charset=utf-8"},
 638                         },
 639                 },
 640                 {
 641                         method:       "GET",
 642                         path:         s.ctrUUID + "/a/b/c/d.html",
 643                         expectStatus: http.StatusOK,
 644                         expectBodyRe: "<html></html>\n",
 645                         expectHeader: http.Header{
 646                                 "Content-Type": {"text/html; charset=utf-8"},
 647                         },
 648                 },
 649         } {
 650                 c.Logf("trial %#v", trial)
 651                 ctx := s.userctx
 652                 if trial.unauthenticated {
 653                         ctx = auth.NewContext(context.Background(), auth.CredentialsFromRequest(&http.Request{URL: &url.URL{}, Header: http.Header{}}))
 654                 }
 655                 r, err := http.NewRequestWithContext(ctx, trial.method, "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+trial.path, nil)
 656                 c.Assert(err, check.IsNil)
 657                 for k := range trial.header {
 658                         r.Header.Set(k, trial.header.Get(k))
 659                 }
 660                 handler, err := s.localdb.ContainerRequestLog(ctx, arvados.ContainerLogOptions{
 661                         UUID: s.reqUUID,
 662                         WebDAVOptions: arvados.WebDAVOptions{
 663                                 Method: trial.method,
 664                                 Header: r.Header,
 665                                 Path:   "/" + trial.path,
 666                         },
 667                 })
 668                 c.Assert(err, check.IsNil)
 669                 c.Assert(handler, check.NotNil)
 670                 rec := httptest.NewRecorder()
 671                 handler.ServeHTTP(rec, r)
 672                 resp := rec.Result()
 673                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
 674                 for k := range trial.expectHeader {
 675                         c.Check(resp.Header[k], check.DeepEquals, trial.expectHeader[k])
 676                 }
 677                 buf, err := ioutil.ReadAll(resp.Body)
 678                 c.Check(err, check.IsNil)
 679                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
 680         }
 681 }
 682
 683 func (s *ContainerGatewaySuite) TestContainerRequestLogViaCadaver(c *check.C) {
 684         s.setupLogCollection(c)
 685
 686         out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "ls")
 687         c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
 688         c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
 689
 690         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
 691         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 692
 693         s.saveLogAndCloseGateway(c)
 694
 695         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
 696         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
 697 }
 698
 699 func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
 700         // Replace s.srv with an HTTP server, otherwise cadaver will
 701         // just fail on TLS cert verification.
 702         s.srv.Close()
 703         rtr := router.New(s.localdb, router.Config{})
 704         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
 705         s.srv.Start()
 706
 707         tempdir, err := ioutil.TempDir("", "localdb-test-")
 708         c.Assert(err, check.IsNil)
 709         defer os.RemoveAll(tempdir)
 710
 711         cmd := exec.Command("cadaver", s.srv.URL+path)
 712         if password != "" {
 713                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
 714                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 715                 c.Assert(err, check.IsNil)
 716                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
 717                 c.Assert(err, check.IsNil)
 718                 c.Assert(f.Close(), check.IsNil)
 719         }
 720         cmd.Stdin = bytes.NewBufferString(stdin)
 721         cmd.Dir = tempdir
 722         stdout, err := cmd.StdoutPipe()
 723         c.Assert(err, check.Equals, nil)
 724         cmd.Stderr = cmd.Stdout
 725         c.Logf("cmd: %v", cmd.Args)
 726         go cmd.Start()
 727
 728         var buf bytes.Buffer
 729         _, err = io.Copy(&buf, stdout)
 730         c.Check(err, check.Equals, nil)
 731         err = cmd.Wait()
 732         c.Check(err, check.Equals, nil)
 733         return buf.String()
 734 }
 735
 736 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
 737         c.Logf("connecting to %s", s.gw.Address)
 738         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 739         c.Assert(err, check.IsNil)
 740         c.Assert(sshconn.Conn, check.NotNil)
 741         defer sshconn.Conn.Close()
 742
 743         done := make(chan struct{})
 744         go func() {
 745                 defer close(done)
 746
 747                 // Receive text banner
 748                 buf := make([]byte, 12)
 749                 _, err := io.ReadFull(sshconn.Conn, buf)
 750                 c.Check(err, check.IsNil)
 751                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 752
 753                 // Send text banner
 754                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 755                 c.Check(err, check.IsNil)
 756
 757                 // Receive binary
 758                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 759                 c.Check(err, check.IsNil)
 760
 761                 // If we can get this far into an SSH handshake...
 762                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 763         }()
 764         select {
 765         case <-done:
 766         case <-time.After(time.Second):
 767                 c.Fail()
 768         }
 769         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 770         c.Check(err, check.IsNil)
 771         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 772 }
 773
 774 func (s *ContainerGatewaySuite) TestConnectFail_NoToken(c *check.C) {
 775         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
 776         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 777         c.Check(err, check.ErrorMatches, `.* 401 .*`)
 778 }
 779
 780 func (s *ContainerGatewaySuite) TestConnectFail_AnonymousToken(c *check.C) {
 781         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
 782         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 783         c.Check(err, check.ErrorMatches, `.* 404 .*`)
 784 }
 785
 786 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
 787         // no AuthSecret
 788         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 789                 UUID: s.ctrUUID,
 790         })
 791         c.Check(err, check.ErrorMatches, `authentication error`)
 792         c.Check(conn.Conn, check.IsNil)
 793
 794         // bogus AuthSecret
 795         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 796                 UUID:       s.ctrUUID,
 797                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 798         })
 799         c.Check(err, check.ErrorMatches, `authentication error`)
 800         c.Check(conn.Conn, check.IsNil)
 801
 802         // good AuthSecret
 803         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
 804                 UUID:       s.ctrUUID,
 805                 AuthSecret: s.gw.AuthSecret,
 806         })
 807         c.Check(err, check.IsNil)
 808         c.Check(conn.Conn, check.NotNil)
 809 }
 810
 811 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
 812         forceProxyForTest = true
 813         s.testConnectThroughTunnel(c, "")
 814 }
 815
 816 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
 817         forceProxyForTest = true
 818         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 819         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
 820 }
 821
 822 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
 823         s.testConnectThroughTunnel(c, "")
 824 }
 825
 826 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
 827         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 828         // Until the tunnel starts up, set gateway_address to a value
 829         // that can't work. We want to ensure the only way we can
 830         // reach the gateway is through the tunnel.
 831         tungw := &crunchrun.Gateway{
 832                 ContainerUUID: s.ctrUUID,
 833                 AuthSecret:    s.gw.AuthSecret,
 834                 Log:           ctxlog.TestLogger(c),
 835                 Target:        crunchrun.GatewayTargetStub{},
 836                 ArvadosClient: s.gw.ArvadosClient,
 837                 UpdateTunnelURL: func(url string) {
 838                         c.Logf("UpdateTunnelURL(%q)", url)
 839                         gwaddr := "tunnel " + url
 840                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 841                                 UUID: s.ctrUUID,
 842                                 Attrs: map[string]interface{}{
 843                                         "gateway_address": gwaddr}})
 844                 },
 845         }
 846         c.Assert(tungw.Start(), check.IsNil)
 847
 848         // We didn't supply an external hostname in the Address field,
 849         // so Start() should assign a local address.
 850         host, _, err := net.SplitHostPort(tungw.Address)
 851         c.Assert(err, check.IsNil)
 852         c.Check(host, check.Equals, "127.0.0.1")
 853
 854         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 855                 UUID: s.ctrUUID,
 856                 Attrs: map[string]interface{}{
 857                         "state": arvados.ContainerStateRunning,
 858                 }})
 859         c.Assert(err, check.IsNil)
 860
 861         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
 862                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 863                 c.Assert(err, check.IsNil)
 864                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
 865                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
 866                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
 867                         break
 868                 }
 869         }
 870         return tungw
 871 }
 872
 873 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
 874         s.setupGatewayWithTunnel(c)
 875         c.Log("connecting to gateway through tunnel")
 876         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
 877         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 878         if expectErrorMatch != "" {
 879                 c.Check(err, check.ErrorMatches, expectErrorMatch)
 880                 return
 881         }
 882         c.Assert(err, check.IsNil)
 883         c.Assert(sshconn.Conn, check.NotNil)
 884         defer sshconn.Conn.Close()
 885
 886         done := make(chan struct{})
 887         go func() {
 888                 defer close(done)
 889
 890                 // Receive text banner
 891                 buf := make([]byte, 12)
 892                 _, err := io.ReadFull(sshconn.Conn, buf)
 893                 c.Check(err, check.IsNil)
 894                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
 895
 896                 // Send text banner
 897                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
 898                 c.Check(err, check.IsNil)
 899
 900                 // Receive binary
 901                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
 902                 c.Check(err, check.IsNil)
 903
 904                 // If we can get this far into an SSH handshake...
 905                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
 906         }()
 907         select {
 908         case <-done:
 909         case <-time.After(time.Second):
 910                 c.Fail()
 911         }
 912         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
 913         c.Check(err, check.IsNil)
 914         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
 915 }