lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/hmac"
  11         "crypto/sha256"
  12         "crypto/tls"
  13         "errors"
  14         "fmt"
  15         "io"
  16         "io/ioutil"
  17         "net"
  18         "net/http"
  19         "net/http/cookiejar"
  20         "net/http/httptest"
  21         "net/url"
  22         "os"
  23         "os/exec"
  24         "path/filepath"
  25         "strconv"
  26         "strings"
  27         "sync"
  28         "sync/atomic"
  29         "time"
  30
  31         "git.arvados.org/arvados.git/lib/controller/router"
  32         "git.arvados.org/arvados.git/lib/controller/rpc"
  33         "git.arvados.org/arvados.git/lib/crunchrun"
  34         "git.arvados.org/arvados.git/lib/ctrlctx"
  35         "git.arvados.org/arvados.git/sdk/go/arvados"
  36         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  37         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  38         "git.arvados.org/arvados.git/sdk/go/auth"
  39         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  40         "git.arvados.org/arvados.git/sdk/go/httpserver"
  41         "git.arvados.org/arvados.git/sdk/go/keepclient"
  42         "golang.org/x/crypto/ssh"
  43         check "gopkg.in/check.v1"
  44 )
  45
  46 var _ = check.Suite(&ContainerGatewaySuite{})
  47
  48 type ContainerGatewaySuite struct {
  49         localdbSuite
  50         containerServices []*httpserver.Server
  51         reqCreateOptions  arvados.CreateOptions
  52         reqUUID           string
  53         ctrUUID           string
  54         srv               *httptest.Server
  55         gw                *crunchrun.Gateway
  56         assignedExtPort   atomic.Int32
  57 }
  58
  59 const (
  60         testDynamicPortMin = 10000
  61         testDynamicPortMax = 20000
  62 )
  63
  64 func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
  65         s.localdbSuite.SetUpSuite(c)
  66
  67         // Set up 10 http servers to play the role of services running
  68         // inside a container. (crunchrun.GatewayTargetStub will allow
  69         // our crunchrun.Gateway to connect to them directly on
  70         // localhost, rather than actually running them inside a
  71         // container.)
  72         for i := 0; i < 10; i++ {
  73                 srv := &httpserver.Server{
  74                         Addr: ":0",
  75                         Server: http.Server{
  76                                 Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  77                                         w.WriteHeader(http.StatusOK)
  78                                         body := fmt.Sprintf("handled %s %s with Host %s", r.Method, r.URL.String(), r.Host)
  79                                         c.Logf("%s", body)
  80                                         w.Write([]byte(body))
  81                                 }),
  82                         },
  83                 }
  84                 srv.Start()
  85                 s.containerServices = append(s.containerServices, srv)
  86         }
  87
  88         // s.containerServices[0] will be unlisted
  89         // s.containerServices[1] will be listed with access=public
  90         // s.containerServices[2,...] will be listed with access=private
  91         publishedPorts := make(map[string]arvados.RequestPublishedPort)
  92         for i, srv := range s.containerServices {
  93                 access := arvados.PublishedPortAccessPrivate
  94                 _, port, _ := net.SplitHostPort(srv.Addr)
  95                 if i == 1 {
  96                         access = arvados.PublishedPortAccessPublic
  97                 }
  98                 if i > 0 {
  99                         publishedPorts[port] = arvados.RequestPublishedPort{
 100                                 Access: access,
 101                                 Label:  "port " + port,
 102                         }
 103                 }
 104         }
 105
 106         s.reqCreateOptions = arvados.CreateOptions{
 107                 Attrs: map[string]interface{}{
 108                         "command":             []string{"echo", time.Now().Format(time.RFC3339Nano)},
 109                         "container_count_max": 1,
 110                         "container_image":     "arvados/apitestfixture:latest",
 111                         "cwd":                 "/tmp",
 112                         "environment":         map[string]string{},
 113                         "output_path":         "/out",
 114                         "priority":            1,
 115                         "state":               arvados.ContainerRequestStateCommitted,
 116                         "mounts": map[string]interface{}{
 117                                 "/out": map[string]interface{}{
 118                                         "kind":     "tmp",
 119                                         "capacity": 1000000,
 120                                 },
 121                         },
 122                         "runtime_constraints": map[string]interface{}{
 123                                 "vcpus": 1,
 124                                 "ram":   2,
 125                         },
 126                         "published_ports": publishedPorts}}
 127 }
 128
 129 func (s *ContainerGatewaySuite) TearDownSuite(c *check.C) {
 130         for _, srv := range s.containerServices {
 131                 go srv.Close()
 132         }
 133         s.containerServices = nil
 134         s.localdbSuite.TearDownSuite(c)
 135 }
 136
 137 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
 138         s.localdbSuite.SetUpTest(c)
 139
 140         s.localdb.cluster.Services.ContainerWebServices.ExternalURL.Host = "*.containers.example.com"
 141         s.localdb.cluster.Services.ContainerWebServices.ExternalPortMin = 0
 142         s.localdb.cluster.Services.ContainerWebServices.ExternalPortMax = 0
 143
 144         cr, err := s.localdb.ContainerRequestCreate(s.userctx, s.reqCreateOptions)
 145         c.Assert(err, check.IsNil)
 146         s.reqUUID = cr.UUID
 147         s.ctrUUID = cr.ContainerUUID
 148
 149         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
 150         fmt.Fprint(h, s.ctrUUID)
 151         authKey := fmt.Sprintf("%x", h.Sum(nil))
 152
 153         rtr := router.New(s.localdb, router.Config{
 154                 ContainerWebServices: &s.localdb.cluster.Services.ContainerWebServices,
 155         })
 156         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
 157         s.srv.StartTLS()
 158         // the test setup doesn't use lib/service so
 159         // service.URLFromContext() returns nothing -- instead, this
 160         // is how we advertise our internal URL and enable
 161         // proxy-to-other-controller mode,
 162         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
 163         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 164         ac := &arvados.Client{
 165                 APIHost:   s.srv.Listener.Addr().String(),
 166                 AuthToken: arvadostest.SystemRootToken,
 167                 Insecure:  true,
 168         }
 169         s.gw = &crunchrun.Gateway{
 170                 ContainerUUID: s.ctrUUID,
 171                 AuthSecret:    authKey,
 172                 Address:       "localhost:0",
 173                 Log:           ctxlog.TestLogger(c),
 174                 Target:        crunchrun.GatewayTargetStub{},
 175                 ArvadosClient: ac,
 176         }
 177         c.Assert(s.gw.Start(), check.IsNil)
 178
 179         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 180         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 181                 UUID: s.ctrUUID,
 182                 Attrs: map[string]interface{}{
 183                         "state": arvados.ContainerStateLocked}})
 184         c.Assert(err, check.IsNil)
 185         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 186                 UUID: s.ctrUUID,
 187                 Attrs: map[string]interface{}{
 188                         "state":           arvados.ContainerStateRunning,
 189                         "gateway_address": s.gw.Address}})
 190         c.Assert(err, check.IsNil)
 191
 192         s.cluster.Containers.ShellAccess.Admin = true
 193         s.cluster.Containers.ShellAccess.User = true
 194         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
 195         c.Check(err, check.IsNil)
 196
 197         s.assignedExtPort.Store(testDynamicPortMin)
 198 }
 199
 200 func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
 201         forceProxyForTest = false
 202         if s.reqUUID != "" {
 203                 _, err := s.localdb.ContainerRequestDelete(s.userctx, arvados.DeleteOptions{UUID: s.reqUUID})
 204                 c.Check(err, check.IsNil)
 205         }
 206         if s.srv != nil {
 207                 s.srv.Close()
 208                 s.srv = nil
 209         }
 210         _, err := s.db.Exec(`delete from container_ports where external_port >= $1 and external_port <= $2`, testDynamicPortMin, testDynamicPortMax)
 211         c.Check(err, check.IsNil)
 212         s.localdbSuite.TearDownTest(c)
 213 }
 214
 215 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 216         for _, trial := range []struct {
 217                 configAdmin bool
 218                 configUser  bool
 219                 sendToken   string
 220                 errorCode   int
 221         }{
 222                 {true, true, arvadostest.ActiveTokenV2, 0},
 223                 {true, false, arvadostest.ActiveTokenV2, 503},
 224                 {false, true, arvadostest.ActiveTokenV2, 0},
 225                 {false, false, arvadostest.ActiveTokenV2, 503},
 226                 {true, true, arvadostest.AdminToken, 0},
 227                 {true, false, arvadostest.AdminToken, 0},
 228                 {false, true, arvadostest.AdminToken, 403},
 229                 {false, false, arvadostest.AdminToken, 503},
 230         } {
 231                 c.Logf("trial %#v", trial)
 232                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 233                 s.cluster.Containers.ShellAccess.User = trial.configUser
 234                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 235                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 236                 if trial.errorCode == 0 {
 237                         if !c.Check(err, check.IsNil) {
 238                                 continue
 239                         }
 240                         if !c.Check(sshconn.Conn, check.NotNil) {
 241                                 continue
 242                         }
 243                         sshconn.Conn.Close()
 244                 } else {
 245                         c.Check(err, check.NotNil)
 246                         err, ok := err.(interface{ HTTPStatus() int })
 247                         if c.Check(ok, check.Equals, true) {
 248                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 249                         }
 250                 }
 251         }
 252 }
 253
 254 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 255         // Set up servers on a few TCP ports
 256         var addrs []string
 257         for i := 0; i < 3; i++ {
 258                 ln, err := net.Listen("tcp", ":0")
 259                 c.Assert(err, check.IsNil)
 260                 defer ln.Close()
 261                 addrs = append(addrs, ln.Addr().String())
 262                 go func() {
 263                         for {
 264                                 conn, err := ln.Accept()
 265                                 if err != nil {
 266                                         return
 267                                 }
 268                                 var gotAddr string
 269                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 270                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 271                                 if gotAddr == ln.Addr().String() {
 272                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 273                                 }
 274                                 conn.Close()
 275                         }
 276                 }()
 277         }
 278
 279         c.Logf("connecting to %s", s.gw.Address)
 280         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 281         c.Assert(err, check.IsNil)
 282         c.Assert(sshconn.Conn, check.NotNil)
 283         defer sshconn.Conn.Close()
 284         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 285                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 286         })
 287         c.Assert(err, check.IsNil)
 288         client := ssh.NewClient(conn, chans, reqs)
 289         for _, expectAddr := range addrs {
 290                 _, port, err := net.SplitHostPort(expectAddr)
 291                 c.Assert(err, check.IsNil)
 292
 293                 c.Logf("trying foo:%s", port)
 294                 {
 295                         conn, err := client.Dial("tcp", "foo:"+port)
 296                         c.Assert(err, check.IsNil)
 297                         conn.SetDeadline(time.Now().Add(time.Second))
 298                         buf, err := ioutil.ReadAll(conn)
 299                         c.Check(err, check.IsNil)
 300                         c.Check(string(buf), check.Equals, "")
 301                 }
 302
 303                 c.Logf("trying localhost:%s", port)
 304                 {
 305                         conn, err := client.Dial("tcp", "localhost:"+port)
 306                         c.Assert(err, check.IsNil)
 307                         conn.SetDeadline(time.Now().Add(time.Second))
 308                         conn.Write([]byte(expectAddr + "\n"))
 309                         var gotAddr string
 310                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 311                         c.Check(gotAddr, check.Equals, expectAddr)
 312                 }
 313         }
 314 }
 315
 316 // Connect to crunch-run container gateway directly, using container
 317 // UUID.
 318 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Direct(c *check.C) {
 319         s.testContainerHTTPProxy(c, s.ctrUUID, s.vhostAndTargetForWildcard)
 320 }
 321
 322 // Connect to crunch-run container gateway directly, using container
 323 // request UUID.
 324 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Direct_ContainerRequestUUID(c *check.C) {
 325         s.testContainerHTTPProxy(c, s.reqUUID, s.vhostAndTargetForWildcard)
 326 }
 327
 328 // Connect through a tunnel terminated at this controller process.
 329 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Tunnel(c *check.C) {
 330         s.gw = s.setupGatewayWithTunnel(c)
 331         s.testContainerHTTPProxy(c, s.ctrUUID, s.vhostAndTargetForWildcard)
 332 }
 333
 334 // Connect through a tunnel terminated at a different controller
 335 // process.
 336 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ProxyTunnel(c *check.C) {
 337         forceProxyForTest = true
 338         s.gw = s.setupGatewayWithTunnel(c)
 339         s.testContainerHTTPProxy(c, s.ctrUUID, s.vhostAndTargetForWildcard)
 340 }
 341
 342 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_DynamicPort(c *check.C) {
 343         s.testContainerHTTPProxy(c, s.ctrUUID, s.vhostAndTargetForDynamicPort)
 344 }
 345
 346 func (s *ContainerGatewaySuite) testContainerHTTPProxy(c *check.C, targetUUID string, vhostAndTargetFunc func(*check.C, string, string) (string, string)) {
 347         testMethods := []string{"GET", "POST", "PATCH", "OPTIONS", "DELETE"}
 348
 349         var wg sync.WaitGroup
 350         for idx, srv := range s.containerServices {
 351                 idx, srv := idx, srv
 352                 wg.Add(1)
 353                 go func() {
 354                         defer wg.Done()
 355                         method := testMethods[idx%len(testMethods)]
 356                         _, port, err := net.SplitHostPort(srv.Addr)
 357                         c.Assert(err, check.IsNil, check.Commentf("%s", srv.Addr))
 358                         vhost, target := vhostAndTargetFunc(c, targetUUID, port)
 359                         comment := check.Commentf("srv.Addr %s => proxy vhost %s, target %s", srv.Addr, vhost, target)
 360                         c.Logf("%s", comment.CheckCommentString())
 361                         req, err := http.NewRequest(method, "https://"+vhost+"/via-"+s.gw.Address, nil)
 362                         c.Assert(err, check.IsNil)
 363                         // Token is already passed to
 364                         // ContainerHTTPProxy() call in s.userctx, but
 365                         // we also need to add an auth cookie to the
 366                         // http request: if the request gets passed
 367                         // through http (see forceProxyForTest), the
 368                         // target router will start with a fresh
 369                         // context and load tokens from the request.
 370                         req.AddCookie(&http.Cookie{
 371                                 Name:  "arvados_api_token",
 372                                 Value: auth.EncodeTokenCookie([]byte(arvadostest.ActiveTokenV2)),
 373                         })
 374                         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 375                                 Target:  target,
 376                                 Request: req,
 377                         })
 378                         c.Assert(err, check.IsNil, comment)
 379                         rw := httptest.NewRecorder()
 380                         handler.ServeHTTP(rw, req)
 381                         resp := rw.Result()
 382                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 383                         if cookie := getCookie(resp, "arvados_container_uuid"); c.Check(cookie, check.NotNil) {
 384                                 c.Check(cookie.Value, check.Equals, s.ctrUUID)
 385                         }
 386                         body, err := io.ReadAll(resp.Body)
 387                         c.Assert(err, check.IsNil)
 388                         c.Check(string(body), check.Matches, `handled `+method+` /via-.* with Host \Q`+vhost+`\E`)
 389                 }()
 390         }
 391         wg.Wait()
 392 }
 393
 394 // Return the virtualhost (in the http request) and opts.Target that
 395 // lib/controller/router.Router will pass to ContainerHTTPProxy() when
 396 // Services.ContainerWebServices.ExternalURL is a wildcard like
 397 // "*.containers.example.com".
 398 func (s *ContainerGatewaySuite) vhostAndTargetForWildcard(c *check.C, targetUUID, targetPort string) (string, string) {
 399         return targetUUID + "-" + targetPort + ".containers.example.com", fmt.Sprintf("%s-%s", targetUUID, targetPort)
 400 }
 401
 402 // Return the virtualhost (in the http request) and opts.Target that
 403 // lib/controller/router.Router will pass to ContainerHTTPProxy() when
 404 // Services.ContainerWebServices.ExternalPortMin and
 405 // Services.ContainerWebServices.ExternalPortMax are positive, and
 406 // Services.ContainerWebServices.ExternalURL is not a wildcard.
 407 func (s *ContainerGatewaySuite) vhostAndTargetForDynamicPort(c *check.C, targetUUID, targetPort string) (string, string) {
 408         exthost := "containers.example.com"
 409         s.localdb.cluster.Services.ContainerWebServices.ExternalURL.Host = exthost
 410         s.localdb.cluster.Services.ContainerWebServices.ExternalPortMin = testDynamicPortMin
 411         s.localdb.cluster.Services.ContainerWebServices.ExternalPortMax = testDynamicPortMax
 412         assignedPort := s.assignedExtPort.Add(1)
 413         _, err := s.db.Exec(`insert into container_ports (external_port, container_uuid, container_port) values ($1, $2, $3)`,
 414                 assignedPort, targetUUID, targetPort)
 415         c.Assert(err, check.IsNil)
 416         return fmt.Sprintf("%s:%d", exthost, assignedPort), fmt.Sprintf(":%d", assignedPort)
 417 }
 418
 419 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_NoToken_Unlisted(c *check.C) {
 420         s.testContainerHTTPProxyError(c, 0, "", s.vhostAndTargetForWildcard, http.StatusUnauthorized)
 421 }
 422
 423 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_NoToken_Private(c *check.C) {
 424         s.testContainerHTTPProxyError(c, 2, "", s.vhostAndTargetForWildcard, http.StatusUnauthorized)
 425 }
 426
 427 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_InvalidToken(c *check.C) {
 428         s.testContainerHTTPProxyError(c, 0, arvadostest.ActiveTokenV2+"bogus", s.vhostAndTargetForWildcard, http.StatusUnauthorized)
 429 }
 430
 431 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_AnonymousToken_Unlisted(c *check.C) {
 432         s.testContainerHTTPProxyError(c, 0, arvadostest.AnonymousToken, s.vhostAndTargetForWildcard, http.StatusNotFound)
 433 }
 434
 435 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_AnonymousToken_Private(c *check.C) {
 436         s.testContainerHTTPProxyError(c, 2, arvadostest.AnonymousToken, s.vhostAndTargetForWildcard, http.StatusNotFound)
 437 }
 438
 439 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_CRsDifferentUsers(c *check.C) {
 440         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 441         cr, err := s.localdb.ContainerRequestCreate(rootctx, s.reqCreateOptions)
 442         defer s.localdb.ContainerRequestDelete(rootctx, arvados.DeleteOptions{UUID: cr.UUID})
 443         c.Assert(err, check.IsNil)
 444         c.Assert(cr.ContainerUUID, check.Equals, s.ctrUUID)
 445         s.testContainerHTTPProxyError(c, 0, arvadostest.ActiveTokenV2, s.vhostAndTargetForWildcard, http.StatusForbidden)
 446 }
 447
 448 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_ContainerNotReadable(c *check.C) {
 449         s.testContainerHTTPProxyError(c, 0, arvadostest.SpectatorToken, s.vhostAndTargetForWildcard, http.StatusNotFound)
 450 }
 451
 452 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_DynamicPort(c *check.C) {
 453         s.testContainerHTTPProxyError(c, 0, arvadostest.SpectatorToken, s.vhostAndTargetForDynamicPort, http.StatusNotFound)
 454 }
 455
 456 func (s *ContainerGatewaySuite) testContainerHTTPProxyError(c *check.C, svcIdx int, token string, vhostAndTargetFunc func(*check.C, string, string) (string, string), expectCode int) {
 457         _, svcPort, err := net.SplitHostPort(s.containerServices[svcIdx].Addr)
 458         c.Assert(err, check.IsNil)
 459         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, token)
 460         vhost, target := vhostAndTargetFunc(c, s.ctrUUID, svcPort)
 461         req, err := http.NewRequest("GET", "https://"+vhost+"/via-"+s.gw.Address, nil)
 462         c.Assert(err, check.IsNil)
 463         _, err = s.localdb.ContainerHTTPProxy(ctx, arvados.ContainerHTTPProxyOptions{
 464                 Target:  target,
 465                 Request: req,
 466         })
 467         c.Check(err, check.NotNil)
 468         var se httpserver.HTTPStatusError
 469         c.Assert(errors.As(err, &se), check.Equals, true)
 470         c.Check(se.HTTPStatus(), check.Equals, expectCode)
 471 }
 472
 473 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_CookieAuth(c *check.C) {
 474         s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "GET", "/foobar")
 475 }
 476
 477 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_CookieAuth_POST(c *check.C) {
 478         s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "POST", "/foobar")
 479 }
 480
 481 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth(c *check.C) {
 482         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 483 }
 484
 485 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_Tunnel(c *check.C) {
 486         s.gw = s.setupGatewayWithTunnel(c)
 487         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 488 }
 489
 490 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_ProxyTunnel(c *check.C) {
 491         forceProxyForTest = true
 492         s.gw = s.setupGatewayWithTunnel(c)
 493         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 494 }
 495
 496 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Anonymous(c *check.C) {
 497         s.testContainerHTTPProxyUsingCurl(c, 1, "", "GET", "/foobar")
 498 }
 499
 500 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Anonymous_OPTIONS(c *check.C) {
 501         s.testContainerHTTPProxyUsingCurl(c, 1, "", "OPTIONS", "/foobar")
 502 }
 503
 504 // Check other query parameters are preserved in the
 505 // redirect-with-cookie.
 506 //
 507 // Note the original request has "?baz&baz&..." and this changes to
 508 // "?baz=&baz=&..." in the redirect location.  We trust the target
 509 // service won't be sensitive to this difference.
 510 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_PreserveQuery(c *check.C) {
 511         body := s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?baz&baz&arvados_api_token="+arvadostest.ActiveTokenV2+"&waz=quux")
 512         c.Check(body, check.Matches, `handled GET /foobar\?baz=&baz=&waz=quux with Host `+s.ctrUUID+`.*`)
 513 }
 514
 515 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Patch(c *check.C) {
 516         body := s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "PATCH", "/foobar")
 517         c.Check(body, check.Matches, `handled PATCH /foobar with Host `+s.ctrUUID+`.*`)
 518 }
 519
 520 // Note there is no particular reason this test needs to use curl.  It
 521 // would make sense to rewrite it to use stdlib instead, as we did
 522 // with other tests in commit
 523 // 16f957acf378cd3384d4b9c6ce844fe3cefa600b.
 524 func (s *ContainerGatewaySuite) testContainerHTTPProxyUsingCurl(c *check.C, svcIdx int, cookietoken, method, path string) string {
 525         _, svcPort, err := net.SplitHostPort(s.containerServices[svcIdx].Addr)
 526         c.Assert(err, check.IsNil)
 527
 528         vhost, err := url.Parse(s.srv.URL)
 529         c.Assert(err, check.IsNil)
 530         controllerHost := vhost.Host
 531         vhost.Host = s.ctrUUID + "-" + svcPort + ".containers.example.com"
 532         target, err := vhost.Parse(path)
 533         c.Assert(err, check.IsNil)
 534
 535         tempdir := c.MkDir()
 536         cmd := exec.Command("curl")
 537         if cookietoken != "" {
 538                 cmd.Args = append(cmd.Args, "--cookie", "arvados_api_token="+string(auth.EncodeTokenCookie([]byte(cookietoken))))
 539         } else {
 540                 cmd.Args = append(cmd.Args, "--cookie-jar", filepath.Join(tempdir, "cookies.txt"))
 541         }
 542         if method != "GET" {
 543                 cmd.Args = append(cmd.Args, "--request", method)
 544         }
 545         cmd.Args = append(cmd.Args, "--silent", "--insecure", "--location", "--connect-to", vhost.Hostname()+":443:"+controllerHost, target.String())
 546         cmd.Dir = tempdir
 547         stdout, err := cmd.StdoutPipe()
 548         c.Assert(err, check.Equals, nil)
 549         cmd.Stderr = cmd.Stdout
 550         c.Logf("cmd: %v", cmd.Args)
 551         go cmd.Start()
 552
 553         var buf bytes.Buffer
 554         _, err = io.Copy(&buf, stdout)
 555         c.Check(err, check.Equals, nil)
 556         err = cmd.Wait()
 557         c.Check(err, check.Equals, nil)
 558         c.Check(buf.String(), check.Matches, `handled `+method+` /.*`)
 559         return buf.String()
 560 }
 561
 562 // See testContainerHTTPProxy_ReusedPort_FollowRedirs().  These
 563 // integration tests check the redirect-with-cookie behavior when a
 564 // request arrives on a dynamically-assigned port and it has cookies
 565 // indicating that the client has previously accessed a different
 566 // container's web services on this same port, i.e., it is susceptible
 567 // to leaking cache/cookie/localstorage data from the previous
 568 // container's service to the current container's service.
 569 type testReusedPortFollowRedirs struct {
 570         svcIdx      int
 571         method      string
 572         querytoken  string
 573         cookietoken string
 574 }
 575
 576 // Reject non-GET requests.  In principle we could 303 them, but in
 577 // the most obvious case (an AJAX request initiated by the previous
 578 // container's web application), delivering the request to the new
 579 // container would surely not be the intended behavior.
 580 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_FollowRedirs_RejectPOST(c *check.C) {
 581         code, body, redirs := s.testContainerHTTPProxy_ReusedPort_FollowRedirs(c, testReusedPortFollowRedirs{
 582                 method:      "POST",
 583                 cookietoken: arvadostest.ActiveTokenV2,
 584         })
 585         c.Check(code, check.Equals, http.StatusGone)
 586         c.Check(body, check.Equals, "")
 587         c.Check(redirs, check.HasLen, 0)
 588 }
 589
 590 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_FollowRedirs_WithoutToken_ClearApplicationCookie(c *check.C) {
 591         code, body, redirs := s.testContainerHTTPProxy_ReusedPort_FollowRedirs(c, testReusedPortFollowRedirs{
 592                 svcIdx:      1,
 593                 method:      "GET",
 594                 cookietoken: arvadostest.ActiveTokenV2,
 595         })
 596         c.Check(code, check.Equals, http.StatusOK)
 597         c.Check(body, check.Matches, `handled GET /foobar with Host containers\.example\.com:\d+`)
 598         c.Check(redirs, check.HasLen, 1)
 599 }
 600
 601 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_FollowRedirs_WithToken_ClearApplicationCookie(c *check.C) {
 602         code, body, redirs := s.testContainerHTTPProxy_ReusedPort_FollowRedirs(c, testReusedPortFollowRedirs{
 603                 method:     "GET",
 604                 querytoken: arvadostest.ActiveTokenV2,
 605         })
 606         c.Check(code, check.Equals, http.StatusOK)
 607         c.Check(body, check.Matches, `handled GET /foobar with Host containers\.example\.com:\d+`)
 608         if c.Check(redirs, check.HasLen, 1) {
 609                 c.Check(redirs[0], check.Matches, `https://containers\.example\.com:\d+/foobar`)
 610         }
 611 }
 612
 613 func (s *ContainerGatewaySuite) testContainerHTTPProxy_ReusedPort_FollowRedirs(c *check.C, t testReusedPortFollowRedirs) (responseCode int, responseBody string, redirectsFollowed []string) {
 614         _, svcPort, err := net.SplitHostPort(s.containerServices[t.svcIdx].Addr)
 615         c.Assert(err, check.IsNil)
 616
 617         srvurl, err := url.Parse(s.srv.URL)
 618         c.Assert(err, check.IsNil)
 619         controllerHost := srvurl.Host
 620
 621         vhost, _ := s.vhostAndTargetForDynamicPort(c, s.ctrUUID, svcPort)
 622         requrl := url.URL{
 623                 Scheme: "https",
 624                 Host:   vhost,
 625                 Path:   "/foobar",
 626         }
 627         if t.querytoken != "" {
 628                 requrl.RawQuery = "arvados_api_token=" + t.querytoken
 629         }
 630
 631         cookies := []*http.Cookie{
 632                 &http.Cookie{Name: "arvados_container_uuid", Value: arvadostest.CompletedContainerUUID},
 633                 &http.Cookie{Name: "stale_cookie", Value: "abcdefghij"},
 634         }
 635         if t.cookietoken != "" {
 636                 cookies = append(cookies, &http.Cookie{Name: "arvados_api_token", Value: string(auth.EncodeTokenCookie([]byte(t.cookietoken)))})
 637         }
 638         jar, err := cookiejar.New(nil)
 639         c.Assert(err, check.IsNil)
 640
 641         client := &http.Client{
 642                 Jar: jar,
 643                 CheckRedirect: func(req *http.Request, via []*http.Request) error {
 644                         redirectsFollowed = append(redirectsFollowed, req.URL.String())
 645                         return nil
 646                 },
 647                 Transport: &http.Transport{
 648                         DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 649                                 return tls.Dial(network, controllerHost, &tls.Config{
 650                                         InsecureSkipVerify: true,
 651                                 })
 652                         },
 653                         TLSClientConfig: &tls.Config{
 654                                 InsecureSkipVerify: true}}}
 655         client.Jar.SetCookies(&url.URL{Scheme: "https", Host: "containers.example.com"}, cookies)
 656         req, err := http.NewRequest(t.method, requrl.String(), nil)
 657         c.Assert(err, check.IsNil)
 658         resp, err := client.Do(req)
 659         c.Assert(err, check.IsNil)
 660         responseCode = resp.StatusCode
 661         body, err := ioutil.ReadAll(resp.Body)
 662         c.Assert(err, check.IsNil)
 663         responseBody = string(body)
 664         if responseCode < 400 {
 665                 for _, cookie := range client.Jar.Cookies(&url.URL{Scheme: "https", Host: "containers.example.com"}) {
 666                         c.Check(cookie.Name, check.Not(check.Equals), "stale_cookie")
 667                         if cookie.Name == "arvados_container_uuid" {
 668                                 c.Check(cookie.Value, check.Not(check.Equals), arvadostest.CompletedContainerUUID)
 669                         }
 670                 }
 671         }
 672         return
 673 }
 674
 675 // Unit tests for clear-cookies-and-redirect behavior when the client
 676 // still has active cookies (and possibly client-side cache) from a
 677 // different container that used to be served on the same
 678 // dynamically-assigned port.
 679 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_QueryToken(c *check.C) {
 680         s.testContainerHTTPProxy_ReusedPort(c, arvadostest.ActiveTokenV2, "")
 681 }
 682 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_CookieToken(c *check.C) {
 683         s.testContainerHTTPProxy_ReusedPort(c, "", arvadostest.ActiveTokenV2)
 684 }
 685 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_NoToken(c *check.C) {
 686         s.testContainerHTTPProxy_ReusedPort(c, "", "")
 687 }
 688 func (s *ContainerGatewaySuite) testContainerHTTPProxy_ReusedPort(c *check.C, querytoken, cookietoken string) {
 689         srv := s.containerServices[0]
 690         method := "GET"
 691         _, port, err := net.SplitHostPort(srv.Addr)
 692         c.Assert(err, check.IsNil, check.Commentf("%s", srv.Addr))
 693         vhost, target := s.vhostAndTargetForDynamicPort(c, s.ctrUUID, port)
 694
 695         var tokenCookie *http.Cookie
 696         if cookietoken != "" {
 697                 tokenCookie = &http.Cookie{
 698                         Name:  "arvados_api_token",
 699                         Value: string(auth.EncodeTokenCookie([]byte(cookietoken))),
 700                 }
 701         }
 702
 703         initialURL := "https://" + vhost + "/via-" + s.gw.Address + "/preserve-path?preserve-param=preserve-value"
 704         if querytoken != "" {
 705                 initialURL += "&arvados_api_token=" + querytoken
 706         }
 707         req, err := http.NewRequest(method, initialURL, nil)
 708         c.Assert(err, check.IsNil)
 709         req.Header.Add("Cookie", "arvados_container_uuid=zzzzz-dz642-compltcontainer")
 710         req.Header.Add("Cookie", "stale_cookie=abcdefghij")
 711         if tokenCookie != nil {
 712                 req.Header.Add("Cookie", tokenCookie.String())
 713         }
 714         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 715                 Target:  target,
 716                 Request: req,
 717         })
 718         c.Assert(err, check.IsNil)
 719         rw := httptest.NewRecorder()
 720         handler.ServeHTTP(rw, req)
 721         resp := rw.Result()
 722         c.Check(resp.StatusCode, check.Equals, http.StatusSeeOther)
 723         c.Logf("Received Location: %s", resp.Header.Get("Location"))
 724         c.Logf("Received cookies: %v", resp.Cookies())
 725         newTokenCookie := getCookie(resp, "arvados_api_token")
 726         if querytoken != "" {
 727                 if c.Check(newTokenCookie, check.NotNil) {
 728                         c.Check(newTokenCookie.Expires.IsZero(), check.Equals, true)
 729                 }
 730         }
 731         if newTokenCookie != nil {
 732                 tokenCookie = newTokenCookie
 733         }
 734         if staleCookie := getCookie(resp, "stale_cookie"); c.Check(staleCookie, check.NotNil) {
 735                 c.Check(staleCookie.Expires.Before(time.Now()), check.Equals, true)
 736                 c.Check(staleCookie.Value, check.Equals, "")
 737         }
 738         if ctrCookie := getCookie(resp, "arvados_container_uuid"); c.Check(ctrCookie, check.NotNil) {
 739                 c.Check(ctrCookie.Expires.Before(time.Now()), check.Equals, true)
 740                 c.Check(ctrCookie.Value, check.Equals, "")
 741         }
 742         c.Check(resp.Header.Get("Clear-Site-Data"), check.Equals, `"cache", "storage"`)
 743
 744         req, err = http.NewRequest(method, resp.Header.Get("Location"), nil)
 745         c.Assert(err, check.IsNil)
 746         req.Header.Add("Cookie", tokenCookie.String())
 747         handler, err = s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 748                 Target:  target,
 749                 Request: req,
 750         })
 751         c.Assert(err, check.IsNil)
 752         rw = httptest.NewRecorder()
 753         handler.ServeHTTP(rw, req)
 754         resp = rw.Result()
 755         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 756         if ctrCookie := getCookie(resp, "arvados_container_uuid"); c.Check(ctrCookie, check.NotNil) {
 757                 c.Check(ctrCookie.Expires.IsZero(), check.Equals, true)
 758                 c.Check(ctrCookie.Value, check.Equals, s.ctrUUID)
 759         }
 760         body, err := ioutil.ReadAll(resp.Body)
 761         c.Check(err, check.IsNil)
 762         c.Check(string(body), check.Matches, `handled GET /via-localhost:\d+/preserve-path\?preserve-param=preserve-value with Host containers.example.com:\d+`)
 763 }
 764
 765 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_PublishedPortByName_ProxyTunnel(c *check.C) {
 766         forceProxyForTest = true
 767         s.gw = s.setupGatewayWithTunnel(c)
 768         s.testContainerHTTPProxy_PublishedPortByName(c)
 769 }
 770
 771 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_PublishedPortByName(c *check.C) {
 772         s.testContainerHTTPProxy_PublishedPortByName(c)
 773 }
 774
 775 func (s *ContainerGatewaySuite) testContainerHTTPProxy_PublishedPortByName(c *check.C) {
 776         srv := s.containerServices[1]
 777         _, port, _ := net.SplitHostPort(srv.Addr)
 778         portnum, err := strconv.Atoi(port)
 779         c.Assert(err, check.IsNil)
 780         namelink, err := s.localdb.LinkCreate(s.userctx, arvados.CreateOptions{
 781                 Attrs: map[string]interface{}{
 782                         "link_class": "published_port",
 783                         "name":       "warthogfacedbuffoon",
 784                         "head_uuid":  s.reqUUID,
 785                         "properties": map[string]interface{}{
 786                                 "port": portnum}}})
 787         c.Assert(err, check.IsNil)
 788         defer s.localdb.LinkDelete(s.userctx, arvados.DeleteOptions{UUID: namelink.UUID})
 789
 790         vhost := namelink.Name + ".containers.example.com"
 791         req, err := http.NewRequest("METHOD", "https://"+vhost+"/path", nil)
 792         c.Assert(err, check.IsNil)
 793         // Token is already passed to ContainerHTTPProxy() call in
 794         // s.userctx, but we also need to add an auth cookie to the
 795         // http request: if the request gets passed through http (see
 796         // forceProxyForTest), the target router will start with a
 797         // fresh context and load tokens from the request.
 798         req.AddCookie(&http.Cookie{
 799                 Name:  "arvados_api_token",
 800                 Value: auth.EncodeTokenCookie([]byte(arvadostest.ActiveTokenV2)),
 801         })
 802         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 803                 Target:  namelink.Name,
 804                 Request: req,
 805         })
 806         c.Assert(err, check.IsNil)
 807         rw := httptest.NewRecorder()
 808         handler.ServeHTTP(rw, req)
 809         resp := rw.Result()
 810         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 811         body, err := io.ReadAll(resp.Body)
 812         c.Assert(err, check.IsNil)
 813         c.Check(string(body), check.Matches, `handled METHOD /path with Host \Q`+vhost+`\E`)
 814 }
 815
 816 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
 817         files := map[string]string{
 818                 "stderr.txt":   "hello world\n",
 819                 "a/b/c/d.html": "<html></html>\n",
 820         }
 821         client := arvados.NewClientFromEnv()
 822         ac, err := arvadosclient.New(client)
 823         c.Assert(err, check.IsNil)
 824         kc, err := keepclient.MakeKeepClient(ac)
 825         c.Assert(err, check.IsNil)
 826         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 827         c.Assert(err, check.IsNil)
 828         for name, content := range files {
 829                 for i, ch := range name {
 830                         if ch == '/' {
 831                                 err := cfs.Mkdir("/"+name[:i], 0777)
 832                                 c.Assert(err, check.IsNil)
 833                         }
 834                 }
 835                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 836                 c.Assert(err, check.IsNil)
 837                 f.Write([]byte(content))
 838                 err = f.Close()
 839                 c.Assert(err, check.IsNil)
 840         }
 841         cfs.Sync()
 842         s.gw.LogCollection = cfs
 843 }
 844
 845 func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
 846         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 847         txt, err := s.gw.LogCollection.MarshalManifest(".")
 848         c.Assert(err, check.IsNil)
 849         coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 850                 Attrs: map[string]interface{}{
 851                         "manifest_text": txt,
 852                 }})
 853         c.Assert(err, check.IsNil)
 854         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 855                 UUID: s.ctrUUID,
 856                 Attrs: map[string]interface{}{
 857                         "state":     arvados.ContainerStateComplete,
 858                         "exit_code": 0,
 859                         "log":       coll.PortableDataHash,
 860                 }})
 861         c.Assert(err, check.IsNil)
 862         updatedReq, err := s.localdb.ContainerRequestGet(rootctx, arvados.GetOptions{UUID: s.reqUUID})
 863         c.Assert(err, check.IsNil)
 864         c.Logf("container request log UUID is %s", updatedReq.LogUUID)
 865         crLog, err := s.localdb.CollectionGet(rootctx, arvados.GetOptions{UUID: updatedReq.LogUUID, Select: []string{"manifest_text"}})
 866         c.Assert(err, check.IsNil)
 867         c.Logf("collection log manifest:\n%s", crLog.ManifestText)
 868         // Ensure localdb can't circumvent the keep-web proxy test by
 869         // getting content from the container gateway.
 870         s.gw.LogCollection = nil
 871 }
 872
 873 func (s *ContainerGatewaySuite) TestContainerRequestLogViaTunnel(c *check.C) {
 874         forceProxyForTest = true
 875         s.gw = s.setupGatewayWithTunnel(c)
 876         s.setupLogCollection(c)
 877
 878         for _, broken := range []bool{false, true} {
 879                 c.Logf("broken=%v", broken)
 880
 881                 if broken {
 882                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 883                 }
 884
 885                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID+"/stderr.txt", nil)
 886                 c.Assert(err, check.IsNil)
 887                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 888                 handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
 889                         UUID: s.reqUUID,
 890                         WebDAVOptions: arvados.WebDAVOptions{
 891                                 Method: "GET",
 892                                 Header: r.Header,
 893                                 Path:   "/" + s.ctrUUID + "/stderr.txt",
 894                         },
 895                 })
 896                 if broken {
 897                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 898                         continue
 899                 }
 900                 c.Check(err, check.IsNil)
 901                 c.Assert(handler, check.NotNil)
 902                 rec := httptest.NewRecorder()
 903                 handler.ServeHTTP(rec, r)
 904                 resp := rec.Result()
 905                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 906                 buf, err := ioutil.ReadAll(resp.Body)
 907                 c.Check(err, check.IsNil)
 908                 c.Check(string(buf), check.Equals, "hello world\n")
 909         }
 910 }
 911
 912 func (s *ContainerGatewaySuite) TestContainerRequestLogViaGateway(c *check.C) {
 913         s.setupLogCollection(c)
 914         s.testContainerRequestLog(c)
 915 }
 916
 917 func (s *ContainerGatewaySuite) TestContainerRequestLogViaKeepWeb(c *check.C) {
 918         s.setupLogCollection(c)
 919         s.saveLogAndCloseGateway(c)
 920         s.testContainerRequestLog(c)
 921 }
 922
 923 func (s *ContainerGatewaySuite) testContainerRequestLog(c *check.C) {
 924         for _, trial := range []struct {
 925                 method          string
 926                 path            string
 927                 header          http.Header
 928                 unauthenticated bool
 929                 expectStatus    int
 930                 expectBodyRe    string
 931                 expectHeader    http.Header
 932         }{
 933                 {
 934                         method:       "GET",
 935                         path:         s.ctrUUID + "/stderr.txt",
 936                         expectStatus: http.StatusOK,
 937                         expectBodyRe: "hello world\n",
 938                         expectHeader: http.Header{
 939                                 "Content-Type": {"text/plain; charset=utf-8"},
 940                         },
 941                 },
 942                 {
 943                         method: "GET",
 944                         path:   s.ctrUUID + "/stderr.txt",
 945                         header: http.Header{
 946                                 "Range": {"bytes=-6"},
 947                         },
 948                         expectStatus: http.StatusPartialContent,
 949                         expectBodyRe: "world\n",
 950                         expectHeader: http.Header{
 951                                 "Content-Type":  {"text/plain; charset=utf-8"},
 952                                 "Content-Range": {"bytes 6-11/12"},
 953                         },
 954                 },
 955                 {
 956                         method:       "OPTIONS",
 957                         path:         s.ctrUUID + "/stderr.txt",
 958                         expectStatus: http.StatusOK,
 959                         expectBodyRe: "",
 960                         expectHeader: http.Header{
 961                                 "Dav":   {"1, 2"},
 962                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 963                         },
 964                 },
 965                 {
 966                         method:          "OPTIONS",
 967                         path:            s.ctrUUID + "/stderr.txt",
 968                         unauthenticated: true,
 969                         header: http.Header{
 970                                 "Access-Control-Request-Method": {"POST"},
 971                         },
 972                         expectStatus: http.StatusOK,
 973                         expectBodyRe: "",
 974                         expectHeader: http.Header{
 975                                 "Access-Control-Allow-Headers": {"Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control"},
 976                                 "Access-Control-Allow-Methods": {"COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK"},
 977                                 "Access-Control-Allow-Origin":  {"*"},
 978                                 "Access-Control-Max-Age":       {"86400"},
 979                         },
 980                 },
 981                 {
 982                         method:       "PROPFIND",
 983                         path:         s.ctrUUID + "/",
 984                         expectStatus: http.StatusMultiStatus,
 985                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 986                         expectHeader: http.Header{
 987                                 "Content-Type": {"text/xml; charset=utf-8"},
 988                         },
 989                 },
 990                 {
 991                         method:       "PROPFIND",
 992                         path:         s.ctrUUID,
 993                         expectStatus: http.StatusMultiStatus,
 994                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 995                         expectHeader: http.Header{
 996                                 "Content-Type": {"text/xml; charset=utf-8"},
 997                         },
 998                 },
 999                 {
1000                         method:       "PROPFIND",
1001                         path:         s.ctrUUID + "/a/b/c/",
1002                         expectStatus: http.StatusMultiStatus,
1003                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*>\n?`,
1004                         expectHeader: http.Header{
1005                                 "Content-Type": {"text/xml; charset=utf-8"},
1006                         },
1007                 },
1008                 {
1009                         method:       "GET",
1010                         path:         s.ctrUUID + "/a/b/c/d.html",
1011                         expectStatus: http.StatusOK,
1012                         expectBodyRe: "<html></html>\n",
1013                         expectHeader: http.Header{
1014                                 "Content-Type": {"text/html; charset=utf-8"},
1015                         },
1016                 },
1017         } {
1018                 c.Logf("trial %#v", trial)
1019                 ctx := s.userctx
1020                 if trial.unauthenticated {
1021                         ctx = auth.NewContext(context.Background(), auth.CredentialsFromRequest(&http.Request{URL: &url.URL{}, Header: http.Header{}}))
1022                 }
1023                 r, err := http.NewRequestWithContext(ctx, trial.method, "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+trial.path, nil)
1024                 c.Assert(err, check.IsNil)
1025                 for k := range trial.header {
1026                         r.Header.Set(k, trial.header.Get(k))
1027                 }
1028                 handler, err := s.localdb.ContainerRequestLog(ctx, arvados.ContainerLogOptions{
1029                         UUID: s.reqUUID,
1030                         WebDAVOptions: arvados.WebDAVOptions{
1031                                 Method: trial.method,
1032                                 Header: r.Header,
1033                                 Path:   "/" + trial.path,
1034                         },
1035                 })
1036                 c.Assert(err, check.IsNil)
1037                 c.Assert(handler, check.NotNil)
1038                 rec := httptest.NewRecorder()
1039                 handler.ServeHTTP(rec, r)
1040                 resp := rec.Result()
1041                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
1042                 for k := range trial.expectHeader {
1043                         c.Check(resp.Header[k], check.DeepEquals, trial.expectHeader[k])
1044                 }
1045                 buf, err := ioutil.ReadAll(resp.Body)
1046                 c.Check(err, check.IsNil)
1047                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
1048         }
1049 }
1050
1051 func (s *ContainerGatewaySuite) TestContainerRequestLogViaCadaver(c *check.C) {
1052         s.setupLogCollection(c)
1053
1054         out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "ls")
1055         c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
1056         c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
1057
1058         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
1059         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
1060
1061         s.saveLogAndCloseGateway(c)
1062
1063         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
1064         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
1065 }
1066
1067 func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
1068         // Replace s.srv with an HTTP server, otherwise cadaver will
1069         // just fail on TLS cert verification.
1070         s.srv.Close()
1071         rtr := router.New(s.localdb, router.Config{})
1072         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
1073         s.srv.Start()
1074
1075         tempdir, err := ioutil.TempDir("", "localdb-test-")
1076         c.Assert(err, check.IsNil)
1077         defer os.RemoveAll(tempdir)
1078
1079         cmd := exec.Command("cadaver", s.srv.URL+path)
1080         if password != "" {
1081                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
1082                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
1083                 c.Assert(err, check.IsNil)
1084                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
1085                 c.Assert(err, check.IsNil)
1086                 c.Assert(f.Close(), check.IsNil)
1087         }
1088         cmd.Stdin = bytes.NewBufferString(stdin)
1089         cmd.Dir = tempdir
1090         stdout, err := cmd.StdoutPipe()
1091         c.Assert(err, check.Equals, nil)
1092         cmd.Stderr = cmd.Stdout
1093         c.Logf("cmd: %v", cmd.Args)
1094         go cmd.Start()
1095
1096         var buf bytes.Buffer
1097         _, err = io.Copy(&buf, stdout)
1098         c.Check(err, check.Equals, nil)
1099         err = cmd.Wait()
1100         c.Check(err, check.Equals, nil)
1101         return buf.String()
1102 }
1103
1104 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
1105         c.Logf("connecting to %s", s.gw.Address)
1106         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
1107         c.Assert(err, check.IsNil)
1108         c.Assert(sshconn.Conn, check.NotNil)
1109         defer sshconn.Conn.Close()
1110
1111         done := make(chan struct{})
1112         go func() {
1113                 defer close(done)
1114
1115                 // Receive text banner
1116                 buf := make([]byte, 12)
1117                 _, err := io.ReadFull(sshconn.Conn, buf)
1118                 c.Check(err, check.IsNil)
1119                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
1120
1121                 // Send text banner
1122                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
1123                 c.Check(err, check.IsNil)
1124
1125                 // Receive binary
1126                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
1127                 c.Check(err, check.IsNil)
1128
1129                 // If we can get this far into an SSH handshake...
1130                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
1131         }()
1132         select {
1133         case <-done:
1134         case <-time.After(time.Second):
1135                 c.Fail()
1136         }
1137         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
1138         c.Check(err, check.IsNil)
1139         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
1140 }
1141
1142 func (s *ContainerGatewaySuite) TestConnectFail_NoToken(c *check.C) {
1143         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
1144         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
1145         c.Check(err, check.ErrorMatches, `.* 401 .*`)
1146 }
1147
1148 func (s *ContainerGatewaySuite) TestConnectFail_AnonymousToken(c *check.C) {
1149         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
1150         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
1151         c.Check(err, check.ErrorMatches, `.* 404 .*`)
1152 }
1153
1154 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
1155         // no AuthSecret
1156         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
1157                 UUID: s.ctrUUID,
1158         })
1159         c.Check(err, check.ErrorMatches, `authentication error`)
1160         c.Check(conn.Conn, check.IsNil)
1161
1162         // bogus AuthSecret
1163         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
1164                 UUID:       s.ctrUUID,
1165                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
1166         })
1167         c.Check(err, check.ErrorMatches, `authentication error`)
1168         c.Check(conn.Conn, check.IsNil)
1169
1170         // good AuthSecret
1171         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
1172                 UUID:       s.ctrUUID,
1173                 AuthSecret: s.gw.AuthSecret,
1174         })
1175         c.Check(err, check.IsNil)
1176         c.Check(conn.Conn, check.NotNil)
1177 }
1178
1179 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
1180         forceProxyForTest = true
1181         s.testConnectThroughTunnel(c, "")
1182 }
1183
1184 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
1185         forceProxyForTest = true
1186         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
1187         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
1188 }
1189
1190 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
1191         s.testConnectThroughTunnel(c, "")
1192 }
1193
1194 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
1195         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
1196         // Until the tunnel starts up, set gateway_address to a value
1197         // that can't work. We want to ensure the only way we can
1198         // reach the gateway is through the tunnel.
1199         tungw := &crunchrun.Gateway{
1200                 ContainerUUID: s.ctrUUID,
1201                 AuthSecret:    s.gw.AuthSecret,
1202                 Log:           ctxlog.TestLogger(c),
1203                 Target:        crunchrun.GatewayTargetStub{},
1204                 ArvadosClient: s.gw.ArvadosClient,
1205                 UpdateTunnelURL: func(url string) {
1206                         c.Logf("UpdateTunnelURL(%q)", url)
1207                         gwaddr := "tunnel " + url
1208                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
1209                                 UUID: s.ctrUUID,
1210                                 Attrs: map[string]interface{}{
1211                                         "gateway_address": gwaddr}})
1212                 },
1213         }
1214         c.Assert(tungw.Start(), check.IsNil)
1215
1216         // We didn't supply an external hostname in the Address field,
1217         // so Start() should assign a local address.
1218         host, _, err := net.SplitHostPort(tungw.Address)
1219         c.Assert(err, check.IsNil)
1220         c.Check(host, check.Equals, "127.0.0.1")
1221
1222         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
1223                 UUID: s.ctrUUID,
1224                 Attrs: map[string]interface{}{
1225                         "state": arvados.ContainerStateRunning,
1226                 }})
1227         c.Assert(err, check.IsNil)
1228
1229         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
1230                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
1231                 c.Assert(err, check.IsNil)
1232                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
1233                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
1234                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
1235                         break
1236                 }
1237         }
1238         return tungw
1239 }
1240
1241 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
1242         s.setupGatewayWithTunnel(c)
1243         c.Log("connecting to gateway through tunnel")
1244         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
1245         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
1246         if expectErrorMatch != "" {
1247                 c.Check(err, check.ErrorMatches, expectErrorMatch)
1248                 return
1249         }
1250         c.Assert(err, check.IsNil)
1251         c.Assert(sshconn.Conn, check.NotNil)
1252         defer sshconn.Conn.Close()
1253
1254         done := make(chan struct{})
1255         go func() {
1256                 defer close(done)
1257
1258                 // Receive text banner
1259                 buf := make([]byte, 12)
1260                 _, err := io.ReadFull(sshconn.Conn, buf)
1261                 c.Check(err, check.IsNil)
1262                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
1263
1264                 // Send text banner
1265                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
1266                 c.Check(err, check.IsNil)
1267
1268                 // Receive binary
1269                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
1270                 c.Check(err, check.IsNil)
1271
1272                 // If we can get this far into an SSH handshake...
1273                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
1274         }()
1275         select {
1276         case <-done:
1277         case <-time.After(time.Second):
1278                 c.Fail()
1279         }
1280         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
1281         c.Check(err, check.IsNil)
1282         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
1283 }
1284
1285 func getCookie(resp *http.Response, name string) *http.Cookie {
1286         for _, cookie := range resp.Cookies() {
1287                 if cookie.Name == name {
1288                         return cookie
1289                 }
1290         }
1291         return nil
1292 }