lib/controller/localdb/container_gateway_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package localdb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/hmac"
  11         "crypto/sha256"
  12         "crypto/tls"
  13         "errors"
  14         "fmt"
  15         "io"
  16         "io/ioutil"
  17         "net"
  18         "net/http"
  19         "net/http/cookiejar"
  20         "net/http/httptest"
  21         "net/url"
  22         "os"
  23         "os/exec"
  24         "path/filepath"
  25         "strconv"
  26         "strings"
  27         "sync"
  28         "sync/atomic"
  29         "time"
  30
  31         "git.arvados.org/arvados.git/lib/controller/router"
  32         "git.arvados.org/arvados.git/lib/controller/rpc"
  33         "git.arvados.org/arvados.git/lib/crunchrun"
  34         "git.arvados.org/arvados.git/lib/ctrlctx"
  35         "git.arvados.org/arvados.git/sdk/go/arvados"
  36         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  37         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  38         "git.arvados.org/arvados.git/sdk/go/auth"
  39         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  40         "git.arvados.org/arvados.git/sdk/go/httpserver"
  41         "git.arvados.org/arvados.git/sdk/go/keepclient"
  42         "golang.org/x/crypto/ssh"
  43         check "gopkg.in/check.v1"
  44 )
  45
  46 var _ = check.Suite(&ContainerGatewaySuite{})
  47
  48 type ContainerGatewaySuite struct {
  49         localdbSuite
  50         containerServices []*httpserver.Server
  51         reqCreateOptions  arvados.CreateOptions
  52         reqUUID           string
  53         ctrUUID           string
  54         srv               *httptest.Server
  55         gw                *crunchrun.Gateway
  56         assignedExtPort   atomic.Int32
  57 }
  58
  59 const (
  60         testDynamicPortMin = 10000
  61         testDynamicPortMax = 20000
  62 )
  63
  64 func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
  65         s.localdbSuite.SetUpSuite(c)
  66
  67         // Set up 10 http servers to play the role of services running
  68         // inside a container. (crunchrun.GatewayTargetStub will allow
  69         // our crunchrun.Gateway to connect to them directly on
  70         // localhost, rather than actually running them inside a
  71         // container.)
  72         for i := 0; i < 10; i++ {
  73                 srv := &httpserver.Server{
  74                         Addr: ":0",
  75                         Server: http.Server{
  76                                 Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  77                                         w.WriteHeader(http.StatusOK)
  78                                         body := fmt.Sprintf("handled %s %s with Host %s", r.Method, r.URL.String(), r.Host)
  79                                         c.Logf("%s", body)
  80                                         w.Write([]byte(body))
  81                                 }),
  82                         },
  83                 }
  84                 srv.Start()
  85                 s.containerServices = append(s.containerServices, srv)
  86         }
  87
  88         // s.containerServices[0] will be unlisted
  89         // s.containerServices[1] will be listed with access=public
  90         // s.containerServices[2,...] will be listed with access=private
  91         publishedPorts := make(map[string]arvados.RequestPublishedPort)
  92         for i, srv := range s.containerServices {
  93                 access := arvados.PublishedPortAccessPrivate
  94                 _, port, _ := net.SplitHostPort(srv.Addr)
  95                 if i == 1 {
  96                         access = arvados.PublishedPortAccessPublic
  97                 }
  98                 if i > 0 {
  99                         publishedPorts[port] = arvados.RequestPublishedPort{
 100                                 Access: access,
 101                                 Label:  "port " + port,
 102                         }
 103                 }
 104         }
 105
 106         s.reqCreateOptions = arvados.CreateOptions{
 107                 Attrs: map[string]interface{}{
 108                         "command":             []string{"echo", time.Now().Format(time.RFC3339Nano)},
 109                         "container_count_max": 1,
 110                         "container_image":     "arvados/apitestfixture:latest",
 111                         "cwd":                 "/tmp",
 112                         "environment":         map[string]string{},
 113                         "output_path":         "/out",
 114                         "priority":            1,
 115                         "state":               arvados.ContainerRequestStateCommitted,
 116                         "mounts": map[string]interface{}{
 117                                 "/out": map[string]interface{}{
 118                                         "kind":     "tmp",
 119                                         "capacity": 1000000,
 120                                 },
 121                         },
 122                         "runtime_constraints": map[string]interface{}{
 123                                 "vcpus": 1,
 124                                 "ram":   2,
 125                         },
 126                         "published_ports": publishedPorts}}
 127 }
 128
 129 func (s *ContainerGatewaySuite) TearDownSuite(c *check.C) {
 130         for _, srv := range s.containerServices {
 131                 go srv.Close()
 132         }
 133         s.containerServices = nil
 134         s.localdbSuite.TearDownSuite(c)
 135 }
 136
 137 func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
 138         s.localdbSuite.SetUpTest(c)
 139
 140         s.localdb.cluster.Services.ContainerWebServices.ExternalURL.Host = "*.containers.example.com"
 141         s.localdb.cluster.Services.ContainerWebServices.ExternalPortMin = 0
 142         s.localdb.cluster.Services.ContainerWebServices.ExternalPortMax = 0
 143
 144         cr, err := s.localdb.ContainerRequestCreate(s.userctx, s.reqCreateOptions)
 145         c.Assert(err, check.IsNil)
 146         s.reqUUID = cr.UUID
 147         s.ctrUUID = cr.ContainerUUID
 148
 149         h := hmac.New(sha256.New, []byte(s.cluster.SystemRootToken))
 150         fmt.Fprint(h, s.ctrUUID)
 151         authKey := fmt.Sprintf("%x", h.Sum(nil))
 152
 153         rtr := router.New(s.localdb, router.Config{
 154                 ContainerWebServices: &s.localdb.cluster.Services.ContainerWebServices,
 155         })
 156         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
 157         s.srv.StartTLS()
 158         // the test setup doesn't use lib/service so
 159         // service.URLFromContext() returns nothing -- instead, this
 160         // is how we advertise our internal URL and enable
 161         // proxy-to-other-controller mode,
 162         forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
 163         s.cluster.Services.Controller.InternalURLs[*forceInternalURLForTest] = arvados.ServiceInstance{}
 164         ac := &arvados.Client{
 165                 APIHost:   s.srv.Listener.Addr().String(),
 166                 AuthToken: arvadostest.SystemRootToken,
 167                 Insecure:  true,
 168         }
 169         s.gw = &crunchrun.Gateway{
 170                 ContainerUUID: s.ctrUUID,
 171                 AuthSecret:    authKey,
 172                 Address:       "localhost:0",
 173                 Log:           ctxlog.TestLogger(c),
 174                 Target:        crunchrun.GatewayTargetStub{},
 175                 ArvadosClient: ac,
 176         }
 177         c.Assert(s.gw.Start(), check.IsNil)
 178
 179         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 180         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 181                 UUID: s.ctrUUID,
 182                 Attrs: map[string]interface{}{
 183                         "state": arvados.ContainerStateLocked}})
 184         c.Assert(err, check.IsNil)
 185         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 186                 UUID: s.ctrUUID,
 187                 Attrs: map[string]interface{}{
 188                         "state":           arvados.ContainerStateRunning,
 189                         "gateway_address": s.gw.Address}})
 190         c.Assert(err, check.IsNil)
 191
 192         s.cluster.Containers.ShellAccess.Admin = true
 193         s.cluster.Containers.ShellAccess.User = true
 194         _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
 195         c.Check(err, check.IsNil)
 196
 197         s.assignedExtPort.Store(testDynamicPortMin)
 198 }
 199
 200 func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
 201         forceProxyForTest = false
 202         if s.reqUUID != "" {
 203                 _, err := s.localdb.ContainerRequestDelete(s.userctx, arvados.DeleteOptions{UUID: s.reqUUID})
 204                 c.Check(err, check.IsNil)
 205         }
 206         if s.srv != nil {
 207                 s.srv.Close()
 208                 s.srv = nil
 209         }
 210         _, err := s.db.Exec(`delete from container_ports where external_port >= $1 and external_port <= $2`, testDynamicPortMin, testDynamicPortMax)
 211         c.Check(err, check.IsNil)
 212         s.localdbSuite.TearDownTest(c)
 213 }
 214
 215 func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
 216         for _, trial := range []struct {
 217                 configAdmin bool
 218                 configUser  bool
 219                 sendToken   string
 220                 errorCode   int
 221         }{
 222                 {true, true, arvadostest.ActiveTokenV2, 0},
 223                 {true, false, arvadostest.ActiveTokenV2, 503},
 224                 {false, true, arvadostest.ActiveTokenV2, 0},
 225                 {false, false, arvadostest.ActiveTokenV2, 503},
 226                 {true, true, arvadostest.AdminToken, 0},
 227                 {true, false, arvadostest.AdminToken, 0},
 228                 {false, true, arvadostest.AdminToken, 403},
 229                 {false, false, arvadostest.AdminToken, 503},
 230         } {
 231                 c.Logf("trial %#v", trial)
 232                 s.cluster.Containers.ShellAccess.Admin = trial.configAdmin
 233                 s.cluster.Containers.ShellAccess.User = trial.configUser
 234                 ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, trial.sendToken)
 235                 sshconn, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 236                 if trial.errorCode == 0 {
 237                         if !c.Check(err, check.IsNil) {
 238                                 continue
 239                         }
 240                         if !c.Check(sshconn.Conn, check.NotNil) {
 241                                 continue
 242                         }
 243                         sshconn.Conn.Close()
 244                 } else {
 245                         c.Check(err, check.NotNil)
 246                         err, ok := err.(interface{ HTTPStatus() int })
 247                         if c.Check(ok, check.Equals, true) {
 248                                 c.Check(err.HTTPStatus(), check.Equals, trial.errorCode)
 249                         }
 250                 }
 251         }
 252 }
 253
 254 func (s *ContainerGatewaySuite) TestDirectTCP(c *check.C) {
 255         // Set up servers on a few TCP ports
 256         var addrs []string
 257         for i := 0; i < 3; i++ {
 258                 ln, err := net.Listen("tcp", ":0")
 259                 c.Assert(err, check.IsNil)
 260                 defer ln.Close()
 261                 addrs = append(addrs, ln.Addr().String())
 262                 go func() {
 263                         for {
 264                                 conn, err := ln.Accept()
 265                                 if err != nil {
 266                                         return
 267                                 }
 268                                 var gotAddr string
 269                                 fmt.Fscanf(conn, "%s\n", &gotAddr)
 270                                 c.Logf("stub server listening at %s received string %q from remote %s", ln.Addr().String(), gotAddr, conn.RemoteAddr())
 271                                 if gotAddr == ln.Addr().String() {
 272                                         fmt.Fprintf(conn, "%s\n", ln.Addr().String())
 273                                 }
 274                                 conn.Close()
 275                         }
 276                 }()
 277         }
 278
 279         c.Logf("connecting to %s", s.gw.Address)
 280         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
 281         c.Assert(err, check.IsNil)
 282         c.Assert(sshconn.Conn, check.NotNil)
 283         defer sshconn.Conn.Close()
 284         conn, chans, reqs, err := ssh.NewClientConn(sshconn.Conn, "zzzz-dz642-abcdeabcdeabcde", &ssh.ClientConfig{
 285                 HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil },
 286         })
 287         c.Assert(err, check.IsNil)
 288         client := ssh.NewClient(conn, chans, reqs)
 289         for _, expectAddr := range addrs {
 290                 _, port, err := net.SplitHostPort(expectAddr)
 291                 c.Assert(err, check.IsNil)
 292
 293                 c.Logf("trying foo:%s", port)
 294                 {
 295                         conn, err := client.Dial("tcp", "foo:"+port)
 296                         c.Assert(err, check.IsNil)
 297                         conn.SetDeadline(time.Now().Add(time.Second))
 298                         buf, err := ioutil.ReadAll(conn)
 299                         c.Check(err, check.IsNil)
 300                         c.Check(string(buf), check.Equals, "")
 301                 }
 302
 303                 c.Logf("trying localhost:%s", port)
 304                 {
 305                         conn, err := client.Dial("tcp", "localhost:"+port)
 306                         c.Assert(err, check.IsNil)
 307                         conn.SetDeadline(time.Now().Add(time.Second))
 308                         conn.Write([]byte(expectAddr + "\n"))
 309                         var gotAddr string
 310                         fmt.Fscanf(conn, "%s\n", &gotAddr)
 311                         c.Check(gotAddr, check.Equals, expectAddr)
 312                 }
 313         }
 314 }
 315
 316 // Connect to crunch-run container gateway directly, using container
 317 // UUID.
 318 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Direct(c *check.C) {
 319         s.testContainerHTTPProxy(c, s.ctrUUID, s.vhostAndTargetForWildcard)
 320 }
 321
 322 // Connect to crunch-run container gateway directly, using container
 323 // request UUID.
 324 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Direct_ContainerRequestUUID(c *check.C) {
 325         s.testContainerHTTPProxy(c, s.reqUUID, s.vhostAndTargetForWildcard)
 326 }
 327
 328 // Connect through a tunnel terminated at this controller process.
 329 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Tunnel(c *check.C) {
 330         s.gw = s.setupGatewayWithTunnel(c)
 331         s.testContainerHTTPProxy(c, s.ctrUUID, s.vhostAndTargetForWildcard)
 332 }
 333
 334 // Connect through a tunnel terminated at a different controller
 335 // process.
 336 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ProxyTunnel(c *check.C) {
 337         forceProxyForTest = true
 338         s.gw = s.setupGatewayWithTunnel(c)
 339         s.testContainerHTTPProxy(c, s.ctrUUID, s.vhostAndTargetForWildcard)
 340 }
 341
 342 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_DynamicPort(c *check.C) {
 343         s.testContainerHTTPProxy(c, s.ctrUUID, s.vhostAndTargetForDynamicPort)
 344 }
 345
 346 func (s *ContainerGatewaySuite) testContainerHTTPProxy(c *check.C, targetUUID string, vhostAndTargetFunc func(*check.C, string, string) (string, string)) {
 347         testMethods := []string{"GET", "POST", "PATCH", "OPTIONS", "DELETE"}
 348
 349         var wg sync.WaitGroup
 350         for idx, srv := range s.containerServices {
 351                 idx, srv := idx, srv
 352                 wg.Add(1)
 353                 go func() {
 354                         defer wg.Done()
 355                         method := testMethods[idx%len(testMethods)]
 356                         _, port, err := net.SplitHostPort(srv.Addr)
 357                         c.Assert(err, check.IsNil, check.Commentf("%s", srv.Addr))
 358                         vhost, target := vhostAndTargetFunc(c, targetUUID, port)
 359                         comment := check.Commentf("srv.Addr %s => proxy vhost %s, target %s", srv.Addr, vhost, target)
 360                         c.Logf("%s", comment.CheckCommentString())
 361                         req, err := http.NewRequest(method, "https://"+vhost+"/via-"+s.gw.Address, nil)
 362                         c.Assert(err, check.IsNil)
 363                         // Token is already passed to
 364                         // ContainerHTTPProxy() call in s.userctx, but
 365                         // we also need to add an auth cookie to the
 366                         // http request: if the request gets passed
 367                         // through http (see forceProxyForTest), the
 368                         // target router will start with a fresh
 369                         // context and load tokens from the request.
 370                         req.AddCookie(&http.Cookie{
 371                                 Name:  "arvados_api_token",
 372                                 Value: auth.EncodeTokenCookie([]byte(arvadostest.ActiveTokenV2)),
 373                         })
 374                         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 375                                 Target:  target,
 376                                 Request: req,
 377                         })
 378                         c.Assert(err, check.IsNil, comment)
 379                         rw := httptest.NewRecorder()
 380                         handler.ServeHTTP(rw, req)
 381                         resp := rw.Result()
 382                         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 383                         if cookie := getCookie(resp, "arvados_container_uuid"); c.Check(cookie, check.NotNil) {
 384                                 c.Check(cookie.Value, check.Equals, s.ctrUUID)
 385                         }
 386                         body, err := io.ReadAll(resp.Body)
 387                         c.Assert(err, check.IsNil)
 388                         c.Check(string(body), check.Matches, `handled `+method+` /via-.* with Host \Q`+vhost+`\E`)
 389                 }()
 390         }
 391         wg.Wait()
 392 }
 393
 394 // Return the virtualhost (in the http request) and opts.Target that
 395 // lib/controller/router.Router will pass to ContainerHTTPProxy() when
 396 // Services.ContainerWebServices.ExternalURL is a wildcard like
 397 // "*.containers.example.com".
 398 func (s *ContainerGatewaySuite) vhostAndTargetForWildcard(c *check.C, targetUUID, targetPort string) (string, string) {
 399         return targetUUID + "-" + targetPort + ".containers.example.com", fmt.Sprintf("%s-%s", targetUUID, targetPort)
 400 }
 401
 402 // Return the virtualhost (in the http request) and opts.Target that
 403 // lib/controller/router.Router will pass to ContainerHTTPProxy() when
 404 // Services.ContainerWebServices.ExternalPortMin and
 405 // Services.ContainerWebServices.ExternalPortMax are positive, and
 406 // Services.ContainerWebServices.ExternalURL is not a wildcard.
 407 func (s *ContainerGatewaySuite) vhostAndTargetForDynamicPort(c *check.C, targetUUID, targetPort string) (string, string) {
 408         exthost := "containers.example.com"
 409         s.localdb.cluster.Services.ContainerWebServices.ExternalURL.Host = exthost
 410         s.localdb.cluster.Services.ContainerWebServices.ExternalPortMin = testDynamicPortMin
 411         s.localdb.cluster.Services.ContainerWebServices.ExternalPortMax = testDynamicPortMax
 412         assignedPort := s.assignedExtPort.Add(1)
 413         _, err := s.db.Exec(`insert into container_ports (external_port, container_uuid, container_port) values ($1, $2, $3)`,
 414                 assignedPort, targetUUID, targetPort)
 415         c.Assert(err, check.IsNil)
 416         return fmt.Sprintf("%s:%d", exthost, assignedPort), fmt.Sprintf(":%d", assignedPort)
 417 }
 418
 419 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_NoToken_Unlisted(c *check.C) {
 420         s.testContainerHTTPProxyError(c, 0, "", s.vhostAndTargetForWildcard, http.StatusUnauthorized)
 421 }
 422
 423 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_NoToken_Private(c *check.C) {
 424         s.testContainerHTTPProxyError(c, 2, "", s.vhostAndTargetForWildcard, http.StatusUnauthorized)
 425 }
 426
 427 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_InvalidToken(c *check.C) {
 428         s.testContainerHTTPProxyError(c, 0, arvadostest.ActiveTokenV2+"bogus", s.vhostAndTargetForWildcard, http.StatusUnauthorized)
 429 }
 430
 431 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_AnonymousToken_Unlisted(c *check.C) {
 432         s.testContainerHTTPProxyError(c, 0, arvadostest.AnonymousToken, s.vhostAndTargetForWildcard, http.StatusNotFound)
 433 }
 434
 435 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_AnonymousToken_Private(c *check.C) {
 436         s.testContainerHTTPProxyError(c, 2, arvadostest.AnonymousToken, s.vhostAndTargetForWildcard, http.StatusNotFound)
 437 }
 438
 439 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_CRsDifferentUsers(c *check.C) {
 440         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 441         cr, err := s.localdb.ContainerRequestCreate(rootctx, s.reqCreateOptions)
 442         defer s.localdb.ContainerRequestDelete(rootctx, arvados.DeleteOptions{UUID: cr.UUID})
 443         c.Assert(err, check.IsNil)
 444         c.Assert(cr.ContainerUUID, check.Equals, s.ctrUUID)
 445         s.testContainerHTTPProxyError(c, 0, arvadostest.ActiveTokenV2, s.vhostAndTargetForWildcard, http.StatusForbidden)
 446 }
 447
 448 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_ContainerNotReadable(c *check.C) {
 449         s.testContainerHTTPProxyError(c, 0, arvadostest.SpectatorToken, s.vhostAndTargetForWildcard, http.StatusNotFound)
 450 }
 451
 452 func (s *ContainerGatewaySuite) TestContainerHTTPProxyError_DynamicPort(c *check.C) {
 453         s.testContainerHTTPProxyError(c, 0, arvadostest.SpectatorToken, s.vhostAndTargetForDynamicPort, http.StatusNotFound)
 454 }
 455
 456 func (s *ContainerGatewaySuite) testContainerHTTPProxyError(c *check.C, svcIdx int, token string, vhostAndTargetFunc func(*check.C, string, string) (string, string), expectCode int) {
 457         _, svcPort, err := net.SplitHostPort(s.containerServices[svcIdx].Addr)
 458         c.Assert(err, check.IsNil)
 459         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, token)
 460         vhost, target := vhostAndTargetFunc(c, s.ctrUUID, svcPort)
 461         req, err := http.NewRequest("GET", "https://"+vhost+"/via-"+s.gw.Address, nil)
 462         c.Assert(err, check.IsNil)
 463         _, err = s.localdb.ContainerHTTPProxy(ctx, arvados.ContainerHTTPProxyOptions{
 464                 Target:  target,
 465                 Request: req,
 466         })
 467         c.Check(err, check.NotNil)
 468         var se httpserver.HTTPStatusError
 469         c.Assert(errors.As(err, &se), check.Equals, true)
 470         c.Check(se.HTTPStatus(), check.Equals, expectCode)
 471 }
 472
 473 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_CookieAuth(c *check.C) {
 474         s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "GET", "/foobar")
 475 }
 476
 477 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_CookieAuth_POST(c *check.C) {
 478         s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "POST", "/foobar")
 479 }
 480
 481 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth(c *check.C) {
 482         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 483 }
 484
 485 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_Tunnel(c *check.C) {
 486         s.gw = s.setupGatewayWithTunnel(c)
 487         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 488 }
 489
 490 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_ProxyTunnel(c *check.C) {
 491         forceProxyForTest = true
 492         s.gw = s.setupGatewayWithTunnel(c)
 493         s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?arvados_api_token="+arvadostest.ActiveTokenV2)
 494 }
 495
 496 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Anonymous(c *check.C) {
 497         s.testContainerHTTPProxyUsingCurl(c, 1, "", "GET", "/foobar")
 498 }
 499
 500 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Anonymous_OPTIONS(c *check.C) {
 501         s.testContainerHTTPProxyUsingCurl(c, 1, "", "OPTIONS", "/foobar")
 502 }
 503
 504 // Check other query parameters are preserved in the
 505 // redirect-with-cookie.
 506 //
 507 // Note the original request has "?baz&baz&..." and this changes to
 508 // "?baz=&baz=&..." in the redirect location.  We trust the target
 509 // service won't be sensitive to this difference.
 510 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_QueryAuth_PreserveQuery(c *check.C) {
 511         body := s.testContainerHTTPProxyUsingCurl(c, 0, "", "GET", "/foobar?baz&baz&arvados_api_token="+arvadostest.ActiveTokenV2+"&waz=quux")
 512         c.Check(body, check.Matches, `handled GET /foobar\?baz=&baz=&waz=quux with Host `+s.ctrUUID+`.*`)
 513 }
 514
 515 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_Curl_Patch(c *check.C) {
 516         body := s.testContainerHTTPProxyUsingCurl(c, 0, arvadostest.ActiveTokenV2, "PATCH", "/foobar")
 517         c.Check(body, check.Matches, `handled PATCH /foobar with Host `+s.ctrUUID+`.*`)
 518 }
 519
 520 func (s *ContainerGatewaySuite) testContainerHTTPProxyUsingCurl(c *check.C, svcIdx int, cookietoken, method, path string) string {
 521         _, svcPort, err := net.SplitHostPort(s.containerServices[svcIdx].Addr)
 522         c.Assert(err, check.IsNil)
 523
 524         vhost, err := url.Parse(s.srv.URL)
 525         c.Assert(err, check.IsNil)
 526         controllerHost := vhost.Host
 527         vhost.Host = s.ctrUUID + "-" + svcPort + ".containers.example.com"
 528         target, err := vhost.Parse(path)
 529         c.Assert(err, check.IsNil)
 530
 531         tempdir := c.MkDir()
 532         cmd := exec.Command("curl")
 533         if cookietoken != "" {
 534                 cmd.Args = append(cmd.Args, "--cookie", "arvados_api_token="+string(auth.EncodeTokenCookie([]byte(cookietoken))))
 535         } else {
 536                 cmd.Args = append(cmd.Args, "--cookie-jar", filepath.Join(tempdir, "cookies.txt"))
 537         }
 538         if method != "GET" {
 539                 cmd.Args = append(cmd.Args, "--request", method)
 540         }
 541         cmd.Args = append(cmd.Args, "--silent", "--insecure", "--location", "--connect-to", vhost.Hostname()+":443:"+controllerHost, target.String())
 542         cmd.Dir = tempdir
 543         stdout, err := cmd.StdoutPipe()
 544         c.Assert(err, check.Equals, nil)
 545         cmd.Stderr = cmd.Stdout
 546         c.Logf("cmd: %v", cmd.Args)
 547         go cmd.Start()
 548
 549         var buf bytes.Buffer
 550         _, err = io.Copy(&buf, stdout)
 551         c.Check(err, check.Equals, nil)
 552         err = cmd.Wait()
 553         c.Check(err, check.Equals, nil)
 554         c.Check(buf.String(), check.Matches, `handled `+method+` /.*`)
 555         return buf.String()
 556 }
 557
 558 // See testContainerHTTPProxy_ReusedPort_FollowRedirs().  These
 559 // integration tests use curl to check the redirect-with-cookie
 560 // behavior when a request arrives on a dynamically-assigned port and
 561 // it has cookies indicating that the client has previously accessed a
 562 // different container's web services on this same port, i.e., it is
 563 // susceptible to leaking cache/cookie/localstorage data from the
 564 // previous container's service to the current container's service.
 565 type testReusedPortCurl struct {
 566         svcIdx      int
 567         method      string
 568         querytoken  string
 569         cookietoken string
 570 }
 571
 572 // Reject non-GET requests.  In principle we could 303 them, but in
 573 // the most obvious case (an AJAX request initiated by the previous
 574 // container's web application), delivering the request to the new
 575 // container would surely not be the intended behavior.
 576 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_FollowRedirs_RejectPOST(c *check.C) {
 577         code, body, redirs := s.testContainerHTTPProxy_ReusedPort_FollowRedirs(c, testReusedPortCurl{
 578                 method:      "POST",
 579                 cookietoken: arvadostest.ActiveTokenV2,
 580         })
 581         c.Check(code, check.Equals, http.StatusGone)
 582         c.Check(body, check.Equals, "")
 583         c.Check(redirs, check.HasLen, 0)
 584 }
 585
 586 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_FollowRedirs_WithoutToken_ClearApplicationCookie(c *check.C) {
 587         code, body, redirs := s.testContainerHTTPProxy_ReusedPort_FollowRedirs(c, testReusedPortCurl{
 588                 svcIdx:      1,
 589                 method:      "GET",
 590                 cookietoken: arvadostest.ActiveTokenV2,
 591         })
 592         c.Check(code, check.Equals, http.StatusOK)
 593         c.Check(body, check.Matches, `handled GET /foobar with Host containers\.example\.com:\d+`)
 594         c.Check(redirs, check.HasLen, 1)
 595 }
 596
 597 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_FollowRedirs_WithToken_ClearApplicationCookie(c *check.C) {
 598         code, body, redirs := s.testContainerHTTPProxy_ReusedPort_FollowRedirs(c, testReusedPortCurl{
 599                 method:     "GET",
 600                 querytoken: arvadostest.ActiveTokenV2,
 601         })
 602         c.Check(code, check.Equals, http.StatusOK)
 603         c.Check(body, check.Matches, `handled GET /foobar with Host containers\.example\.com:\d+`)
 604         if c.Check(redirs, check.HasLen, 1) {
 605                 c.Check(redirs[0], check.Matches, `https://containers\.example\.com:\d+/foobar`)
 606         }
 607 }
 608
 609 func (s *ContainerGatewaySuite) testContainerHTTPProxy_ReusedPort_FollowRedirs(c *check.C, t testReusedPortCurl) (responseCode int, responseBody string, redirectsFollowed []string) {
 610         _, svcPort, err := net.SplitHostPort(s.containerServices[t.svcIdx].Addr)
 611         c.Assert(err, check.IsNil)
 612
 613         srvurl, err := url.Parse(s.srv.URL)
 614         c.Assert(err, check.IsNil)
 615         controllerHost := srvurl.Host
 616
 617         vhost, _ := s.vhostAndTargetForDynamicPort(c, s.ctrUUID, svcPort)
 618         requrl := url.URL{
 619                 Scheme: "https",
 620                 Host:   vhost,
 621                 Path:   "/foobar",
 622         }
 623         if t.querytoken != "" {
 624                 requrl.RawQuery = "arvados_api_token=" + t.querytoken
 625         }
 626
 627         cookies := []*http.Cookie{
 628                 &http.Cookie{Name: "arvados_container_uuid", Value: arvadostest.CompletedContainerUUID},
 629                 &http.Cookie{Name: "stale_cookie", Value: "abcdefghij"},
 630         }
 631         if t.cookietoken != "" {
 632                 cookies = append(cookies, &http.Cookie{Name: "arvados_api_token", Value: string(auth.EncodeTokenCookie([]byte(t.cookietoken)))})
 633         }
 634         jar, err := cookiejar.New(nil)
 635         c.Assert(err, check.IsNil)
 636
 637         client := &http.Client{
 638                 Jar: jar,
 639                 CheckRedirect: func(req *http.Request, via []*http.Request) error {
 640                         redirectsFollowed = append(redirectsFollowed, req.URL.String())
 641                         return nil
 642                 },
 643                 Transport: &http.Transport{
 644                         DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
 645                                 return tls.Dial(network, controllerHost, &tls.Config{
 646                                         InsecureSkipVerify: true,
 647                                 })
 648                         },
 649                         TLSClientConfig: &tls.Config{
 650                                 InsecureSkipVerify: true}}}
 651         client.Jar.SetCookies(&url.URL{Scheme: "https", Host: "containers.example.com"}, cookies)
 652         req, err := http.NewRequest(t.method, requrl.String(), nil)
 653         c.Assert(err, check.IsNil)
 654         resp, err := client.Do(req)
 655         c.Assert(err, check.IsNil)
 656         responseCode = resp.StatusCode
 657         body, err := ioutil.ReadAll(resp.Body)
 658         c.Assert(err, check.IsNil)
 659         responseBody = string(body)
 660         if responseCode < 400 {
 661                 for _, cookie := range client.Jar.Cookies(&url.URL{Scheme: "https", Host: "containers.example.com"}) {
 662                         c.Check(cookie.Name, check.Not(check.Equals), "stale_cookie")
 663                         if cookie.Name == "arvados_container_uuid" {
 664                                 c.Check(cookie.Value, check.Not(check.Equals), arvadostest.CompletedContainerUUID)
 665                         }
 666                 }
 667         }
 668         return
 669 }
 670
 671 // Unit tests for clear-cookies-and-redirect behavior when the client
 672 // still has active cookies (and possibly client-side cache) from a
 673 // different container that used to be served on the same
 674 // dynamically-assigned port.
 675 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_QueryToken(c *check.C) {
 676         s.testContainerHTTPProxy_ReusedPort(c, arvadostest.ActiveTokenV2, "")
 677 }
 678 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_CookieToken(c *check.C) {
 679         s.testContainerHTTPProxy_ReusedPort(c, "", arvadostest.ActiveTokenV2)
 680 }
 681 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_ReusedPort_NoToken(c *check.C) {
 682         s.testContainerHTTPProxy_ReusedPort(c, "", "")
 683 }
 684 func (s *ContainerGatewaySuite) testContainerHTTPProxy_ReusedPort(c *check.C, querytoken, cookietoken string) {
 685         srv := s.containerServices[0]
 686         method := "GET"
 687         _, port, err := net.SplitHostPort(srv.Addr)
 688         c.Assert(err, check.IsNil, check.Commentf("%s", srv.Addr))
 689         vhost, target := s.vhostAndTargetForDynamicPort(c, s.ctrUUID, port)
 690
 691         var tokenCookie *http.Cookie
 692         if cookietoken != "" {
 693                 tokenCookie = &http.Cookie{
 694                         Name:  "arvados_api_token",
 695                         Value: string(auth.EncodeTokenCookie([]byte(cookietoken))),
 696                 }
 697         }
 698
 699         initialURL := "https://" + vhost + "/via-" + s.gw.Address + "/preserve-path?preserve-param=preserve-value"
 700         if querytoken != "" {
 701                 initialURL += "&arvados_api_token=" + querytoken
 702         }
 703         req, err := http.NewRequest(method, initialURL, nil)
 704         c.Assert(err, check.IsNil)
 705         req.Header.Add("Cookie", "arvados_container_uuid=zzzzz-dz642-compltcontainer")
 706         req.Header.Add("Cookie", "stale_cookie=abcdefghij")
 707         if tokenCookie != nil {
 708                 req.Header.Add("Cookie", tokenCookie.String())
 709         }
 710         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 711                 Target:  target,
 712                 Request: req,
 713         })
 714         c.Assert(err, check.IsNil)
 715         rw := httptest.NewRecorder()
 716         handler.ServeHTTP(rw, req)
 717         resp := rw.Result()
 718         c.Check(resp.StatusCode, check.Equals, http.StatusSeeOther)
 719         c.Logf("Received Location: %s", resp.Header.Get("Location"))
 720         c.Logf("Received cookies: %v", resp.Cookies())
 721         newTokenCookie := getCookie(resp, "arvados_api_token")
 722         if querytoken != "" {
 723                 if c.Check(newTokenCookie, check.NotNil) {
 724                         c.Check(newTokenCookie.Expires.IsZero(), check.Equals, true)
 725                 }
 726         }
 727         if newTokenCookie != nil {
 728                 tokenCookie = newTokenCookie
 729         }
 730         if staleCookie := getCookie(resp, "stale_cookie"); c.Check(staleCookie, check.NotNil) {
 731                 c.Check(staleCookie.Expires.Before(time.Now()), check.Equals, true)
 732                 c.Check(staleCookie.Value, check.Equals, "")
 733         }
 734         if ctrCookie := getCookie(resp, "arvados_container_uuid"); c.Check(ctrCookie, check.NotNil) {
 735                 c.Check(ctrCookie.Expires.Before(time.Now()), check.Equals, true)
 736                 c.Check(ctrCookie.Value, check.Equals, "")
 737         }
 738         c.Check(resp.Header.Get("Clear-Site-Data"), check.Equals, `"cache", "storage"`)
 739
 740         req, err = http.NewRequest(method, resp.Header.Get("Location"), nil)
 741         c.Assert(err, check.IsNil)
 742         req.Header.Add("Cookie", tokenCookie.String())
 743         handler, err = s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 744                 Target:  target,
 745                 Request: req,
 746         })
 747         c.Assert(err, check.IsNil)
 748         rw = httptest.NewRecorder()
 749         handler.ServeHTTP(rw, req)
 750         resp = rw.Result()
 751         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 752         if ctrCookie := getCookie(resp, "arvados_container_uuid"); c.Check(ctrCookie, check.NotNil) {
 753                 c.Check(ctrCookie.Expires.IsZero(), check.Equals, true)
 754                 c.Check(ctrCookie.Value, check.Equals, s.ctrUUID)
 755         }
 756         body, err := ioutil.ReadAll(resp.Body)
 757         c.Check(err, check.IsNil)
 758         c.Check(string(body), check.Matches, `handled GET /via-localhost:\d+/preserve-path\?preserve-param=preserve-value with Host containers.example.com:\d+`)
 759 }
 760
 761 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_PublishedPortByName_ProxyTunnel(c *check.C) {
 762         forceProxyForTest = true
 763         s.gw = s.setupGatewayWithTunnel(c)
 764         s.testContainerHTTPProxy_PublishedPortByName(c)
 765 }
 766
 767 func (s *ContainerGatewaySuite) TestContainerHTTPProxy_PublishedPortByName(c *check.C) {
 768         s.testContainerHTTPProxy_PublishedPortByName(c)
 769 }
 770
 771 func (s *ContainerGatewaySuite) testContainerHTTPProxy_PublishedPortByName(c *check.C) {
 772         srv := s.containerServices[1]
 773         _, port, _ := net.SplitHostPort(srv.Addr)
 774         portnum, err := strconv.Atoi(port)
 775         c.Assert(err, check.IsNil)
 776         namelink, err := s.localdb.LinkCreate(s.userctx, arvados.CreateOptions{
 777                 Attrs: map[string]interface{}{
 778                         "link_class": "published_port",
 779                         "name":       "warthogfacedbuffoon",
 780                         "head_uuid":  s.reqUUID,
 781                         "properties": map[string]interface{}{
 782                                 "port": portnum}}})
 783         c.Assert(err, check.IsNil)
 784         defer s.localdb.LinkDelete(s.userctx, arvados.DeleteOptions{UUID: namelink.UUID})
 785
 786         vhost := namelink.Name + ".containers.example.com"
 787         req, err := http.NewRequest("METHOD", "https://"+vhost+"/path", nil)
 788         c.Assert(err, check.IsNil)
 789         // Token is already passed to ContainerHTTPProxy() call in
 790         // s.userctx, but we also need to add an auth cookie to the
 791         // http request: if the request gets passed through http (see
 792         // forceProxyForTest), the target router will start with a
 793         // fresh context and load tokens from the request.
 794         req.AddCookie(&http.Cookie{
 795                 Name:  "arvados_api_token",
 796                 Value: auth.EncodeTokenCookie([]byte(arvadostest.ActiveTokenV2)),
 797         })
 798         handler, err := s.localdb.ContainerHTTPProxy(s.userctx, arvados.ContainerHTTPProxyOptions{
 799                 Target:  namelink.Name,
 800                 Request: req,
 801         })
 802         c.Assert(err, check.IsNil)
 803         rw := httptest.NewRecorder()
 804         handler.ServeHTTP(rw, req)
 805         resp := rw.Result()
 806         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 807         body, err := io.ReadAll(resp.Body)
 808         c.Assert(err, check.IsNil)
 809         c.Check(string(body), check.Matches, `handled METHOD /path with Host \Q`+vhost+`\E`)
 810 }
 811
 812 func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
 813         files := map[string]string{
 814                 "stderr.txt":   "hello world\n",
 815                 "a/b/c/d.html": "<html></html>\n",
 816         }
 817         client := arvados.NewClientFromEnv()
 818         ac, err := arvadosclient.New(client)
 819         c.Assert(err, check.IsNil)
 820         kc, err := keepclient.MakeKeepClient(ac)
 821         c.Assert(err, check.IsNil)
 822         cfs, err := (&arvados.Collection{}).FileSystem(client, kc)
 823         c.Assert(err, check.IsNil)
 824         for name, content := range files {
 825                 for i, ch := range name {
 826                         if ch == '/' {
 827                                 err := cfs.Mkdir("/"+name[:i], 0777)
 828                                 c.Assert(err, check.IsNil)
 829                         }
 830                 }
 831                 f, err := cfs.OpenFile("/"+name, os.O_CREATE|os.O_WRONLY, 0777)
 832                 c.Assert(err, check.IsNil)
 833                 f.Write([]byte(content))
 834                 err = f.Close()
 835                 c.Assert(err, check.IsNil)
 836         }
 837         cfs.Sync()
 838         s.gw.LogCollection = cfs
 839 }
 840
 841 func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
 842         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
 843         txt, err := s.gw.LogCollection.MarshalManifest(".")
 844         c.Assert(err, check.IsNil)
 845         coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
 846                 Attrs: map[string]interface{}{
 847                         "manifest_text": txt,
 848                 }})
 849         c.Assert(err, check.IsNil)
 850         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
 851                 UUID: s.ctrUUID,
 852                 Attrs: map[string]interface{}{
 853                         "state":     arvados.ContainerStateComplete,
 854                         "exit_code": 0,
 855                         "log":       coll.PortableDataHash,
 856                 }})
 857         c.Assert(err, check.IsNil)
 858         updatedReq, err := s.localdb.ContainerRequestGet(rootctx, arvados.GetOptions{UUID: s.reqUUID})
 859         c.Assert(err, check.IsNil)
 860         c.Logf("container request log UUID is %s", updatedReq.LogUUID)
 861         crLog, err := s.localdb.CollectionGet(rootctx, arvados.GetOptions{UUID: updatedReq.LogUUID, Select: []string{"manifest_text"}})
 862         c.Assert(err, check.IsNil)
 863         c.Logf("collection log manifest:\n%s", crLog.ManifestText)
 864         // Ensure localdb can't circumvent the keep-web proxy test by
 865         // getting content from the container gateway.
 866         s.gw.LogCollection = nil
 867 }
 868
 869 func (s *ContainerGatewaySuite) TestContainerRequestLogViaTunnel(c *check.C) {
 870         forceProxyForTest = true
 871         s.gw = s.setupGatewayWithTunnel(c)
 872         s.setupLogCollection(c)
 873
 874         for _, broken := range []bool{false, true} {
 875                 c.Logf("broken=%v", broken)
 876
 877                 if broken {
 878                         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
 879                 }
 880
 881                 r, err := http.NewRequestWithContext(s.userctx, "GET", "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID+"/stderr.txt", nil)
 882                 c.Assert(err, check.IsNil)
 883                 r.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 884                 handler, err := s.localdb.ContainerRequestLog(s.userctx, arvados.ContainerLogOptions{
 885                         UUID: s.reqUUID,
 886                         WebDAVOptions: arvados.WebDAVOptions{
 887                                 Method: "GET",
 888                                 Header: r.Header,
 889                                 Path:   "/" + s.ctrUUID + "/stderr.txt",
 890                         },
 891                 })
 892                 if broken {
 893                         c.Check(err, check.ErrorMatches, `.*tunnel endpoint is invalid.*`)
 894                         continue
 895                 }
 896                 c.Check(err, check.IsNil)
 897                 c.Assert(handler, check.NotNil)
 898                 rec := httptest.NewRecorder()
 899                 handler.ServeHTTP(rec, r)
 900                 resp := rec.Result()
 901                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 902                 buf, err := ioutil.ReadAll(resp.Body)
 903                 c.Check(err, check.IsNil)
 904                 c.Check(string(buf), check.Equals, "hello world\n")
 905         }
 906 }
 907
 908 func (s *ContainerGatewaySuite) TestContainerRequestLogViaGateway(c *check.C) {
 909         s.setupLogCollection(c)
 910         s.testContainerRequestLog(c)
 911 }
 912
 913 func (s *ContainerGatewaySuite) TestContainerRequestLogViaKeepWeb(c *check.C) {
 914         s.setupLogCollection(c)
 915         s.saveLogAndCloseGateway(c)
 916         s.testContainerRequestLog(c)
 917 }
 918
 919 func (s *ContainerGatewaySuite) testContainerRequestLog(c *check.C) {
 920         for _, trial := range []struct {
 921                 method          string
 922                 path            string
 923                 header          http.Header
 924                 unauthenticated bool
 925                 expectStatus    int
 926                 expectBodyRe    string
 927                 expectHeader    http.Header
 928         }{
 929                 {
 930                         method:       "GET",
 931                         path:         s.ctrUUID + "/stderr.txt",
 932                         expectStatus: http.StatusOK,
 933                         expectBodyRe: "hello world\n",
 934                         expectHeader: http.Header{
 935                                 "Content-Type": {"text/plain; charset=utf-8"},
 936                         },
 937                 },
 938                 {
 939                         method: "GET",
 940                         path:   s.ctrUUID + "/stderr.txt",
 941                         header: http.Header{
 942                                 "Range": {"bytes=-6"},
 943                         },
 944                         expectStatus: http.StatusPartialContent,
 945                         expectBodyRe: "world\n",
 946                         expectHeader: http.Header{
 947                                 "Content-Type":  {"text/plain; charset=utf-8"},
 948                                 "Content-Range": {"bytes 6-11/12"},
 949                         },
 950                 },
 951                 {
 952                         method:       "OPTIONS",
 953                         path:         s.ctrUUID + "/stderr.txt",
 954                         expectStatus: http.StatusOK,
 955                         expectBodyRe: "",
 956                         expectHeader: http.Header{
 957                                 "Dav":   {"1, 2"},
 958                                 "Allow": {"OPTIONS, LOCK, GET, HEAD, POST, DELETE, PROPPATCH, COPY, MOVE, UNLOCK, PROPFIND, PUT"},
 959                         },
 960                 },
 961                 {
 962                         method:          "OPTIONS",
 963                         path:            s.ctrUUID + "/stderr.txt",
 964                         unauthenticated: true,
 965                         header: http.Header{
 966                                 "Access-Control-Request-Method": {"POST"},
 967                         },
 968                         expectStatus: http.StatusOK,
 969                         expectBodyRe: "",
 970                         expectHeader: http.Header{
 971                                 "Access-Control-Allow-Headers": {"Authorization, Content-Type, Range, Depth, Destination, If, Lock-Token, Overwrite, Timeout, Cache-Control"},
 972                                 "Access-Control-Allow-Methods": {"COPY, DELETE, GET, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, RMCOL, UNLOCK"},
 973                                 "Access-Control-Allow-Origin":  {"*"},
 974                                 "Access-Control-Max-Age":       {"86400"},
 975                         },
 976                 },
 977                 {
 978                         method:       "PROPFIND",
 979                         path:         s.ctrUUID + "/",
 980                         expectStatus: http.StatusMultiStatus,
 981                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 982                         expectHeader: http.Header{
 983                                 "Content-Type": {"text/xml; charset=utf-8"},
 984                         },
 985                 },
 986                 {
 987                         method:       "PROPFIND",
 988                         path:         s.ctrUUID,
 989                         expectStatus: http.StatusMultiStatus,
 990                         expectBodyRe: `.*\Q<D:displayname>stderr.txt</D:displayname>\E.*>\n?`,
 991                         expectHeader: http.Header{
 992                                 "Content-Type": {"text/xml; charset=utf-8"},
 993                         },
 994                 },
 995                 {
 996                         method:       "PROPFIND",
 997                         path:         s.ctrUUID + "/a/b/c/",
 998                         expectStatus: http.StatusMultiStatus,
 999                         expectBodyRe: `.*\Q<D:displayname>d.html</D:displayname>\E.*>\n?`,
1000                         expectHeader: http.Header{
1001                                 "Content-Type": {"text/xml; charset=utf-8"},
1002                         },
1003                 },
1004                 {
1005                         method:       "GET",
1006                         path:         s.ctrUUID + "/a/b/c/d.html",
1007                         expectStatus: http.StatusOK,
1008                         expectBodyRe: "<html></html>\n",
1009                         expectHeader: http.Header{
1010                                 "Content-Type": {"text/html; charset=utf-8"},
1011                         },
1012                 },
1013         } {
1014                 c.Logf("trial %#v", trial)
1015                 ctx := s.userctx
1016                 if trial.unauthenticated {
1017                         ctx = auth.NewContext(context.Background(), auth.CredentialsFromRequest(&http.Request{URL: &url.URL{}, Header: http.Header{}}))
1018                 }
1019                 r, err := http.NewRequestWithContext(ctx, trial.method, "https://controller.example/arvados/v1/container_requests/"+s.reqUUID+"/log/"+trial.path, nil)
1020                 c.Assert(err, check.IsNil)
1021                 for k := range trial.header {
1022                         r.Header.Set(k, trial.header.Get(k))
1023                 }
1024                 handler, err := s.localdb.ContainerRequestLog(ctx, arvados.ContainerLogOptions{
1025                         UUID: s.reqUUID,
1026                         WebDAVOptions: arvados.WebDAVOptions{
1027                                 Method: trial.method,
1028                                 Header: r.Header,
1029                                 Path:   "/" + trial.path,
1030                         },
1031                 })
1032                 c.Assert(err, check.IsNil)
1033                 c.Assert(handler, check.NotNil)
1034                 rec := httptest.NewRecorder()
1035                 handler.ServeHTTP(rec, r)
1036                 resp := rec.Result()
1037                 c.Check(resp.StatusCode, check.Equals, trial.expectStatus)
1038                 for k := range trial.expectHeader {
1039                         c.Check(resp.Header[k], check.DeepEquals, trial.expectHeader[k])
1040                 }
1041                 buf, err := ioutil.ReadAll(resp.Body)
1042                 c.Check(err, check.IsNil)
1043                 c.Check(string(buf), check.Matches, trial.expectBodyRe)
1044         }
1045 }
1046
1047 func (s *ContainerGatewaySuite) TestContainerRequestLogViaCadaver(c *check.C) {
1048         s.setupLogCollection(c)
1049
1050         out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "ls")
1051         c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
1052         c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
1053
1054         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
1055         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
1056
1057         s.saveLogAndCloseGateway(c)
1058
1059         out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/container_requests/"+s.reqUUID+"/log/"+s.ctrUUID, "get stderr.txt")
1060         c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
1061 }
1062
1063 func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
1064         // Replace s.srv with an HTTP server, otherwise cadaver will
1065         // just fail on TLS cert verification.
1066         s.srv.Close()
1067         rtr := router.New(s.localdb, router.Config{})
1068         s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
1069         s.srv.Start()
1070
1071         tempdir, err := ioutil.TempDir("", "localdb-test-")
1072         c.Assert(err, check.IsNil)
1073         defer os.RemoveAll(tempdir)
1074
1075         cmd := exec.Command("cadaver", s.srv.URL+path)
1076         if password != "" {
1077                 cmd.Env = append(os.Environ(), "HOME="+tempdir)
1078                 f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
1079                 c.Assert(err, check.IsNil)
1080                 _, err = fmt.Fprintf(f, "default login none password %s\n", password)
1081                 c.Assert(err, check.IsNil)
1082                 c.Assert(f.Close(), check.IsNil)
1083         }
1084         cmd.Stdin = bytes.NewBufferString(stdin)
1085         cmd.Dir = tempdir
1086         stdout, err := cmd.StdoutPipe()
1087         c.Assert(err, check.Equals, nil)
1088         cmd.Stderr = cmd.Stdout
1089         c.Logf("cmd: %v", cmd.Args)
1090         go cmd.Start()
1091
1092         var buf bytes.Buffer
1093         _, err = io.Copy(&buf, stdout)
1094         c.Check(err, check.Equals, nil)
1095         err = cmd.Wait()
1096         c.Check(err, check.Equals, nil)
1097         return buf.String()
1098 }
1099
1100 func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
1101         c.Logf("connecting to %s", s.gw.Address)
1102         sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
1103         c.Assert(err, check.IsNil)
1104         c.Assert(sshconn.Conn, check.NotNil)
1105         defer sshconn.Conn.Close()
1106
1107         done := make(chan struct{})
1108         go func() {
1109                 defer close(done)
1110
1111                 // Receive text banner
1112                 buf := make([]byte, 12)
1113                 _, err := io.ReadFull(sshconn.Conn, buf)
1114                 c.Check(err, check.IsNil)
1115                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
1116
1117                 // Send text banner
1118                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
1119                 c.Check(err, check.IsNil)
1120
1121                 // Receive binary
1122                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
1123                 c.Check(err, check.IsNil)
1124
1125                 // If we can get this far into an SSH handshake...
1126                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
1127         }()
1128         select {
1129         case <-done:
1130         case <-time.After(time.Second):
1131                 c.Fail()
1132         }
1133         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
1134         c.Check(err, check.IsNil)
1135         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
1136 }
1137
1138 func (s *ContainerGatewaySuite) TestConnectFail_NoToken(c *check.C) {
1139         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, "")
1140         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
1141         c.Check(err, check.ErrorMatches, `.* 401 .*`)
1142 }
1143
1144 func (s *ContainerGatewaySuite) TestConnectFail_AnonymousToken(c *check.C) {
1145         ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, arvadostest.AnonymousToken)
1146         _, err := s.localdb.ContainerSSH(ctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
1147         c.Check(err, check.ErrorMatches, `.* 404 .*`)
1148 }
1149
1150 func (s *ContainerGatewaySuite) TestCreateTunnel(c *check.C) {
1151         // no AuthSecret
1152         conn, err := s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
1153                 UUID: s.ctrUUID,
1154         })
1155         c.Check(err, check.ErrorMatches, `authentication error`)
1156         c.Check(conn.Conn, check.IsNil)
1157
1158         // bogus AuthSecret
1159         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
1160                 UUID:       s.ctrUUID,
1161                 AuthSecret: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
1162         })
1163         c.Check(err, check.ErrorMatches, `authentication error`)
1164         c.Check(conn.Conn, check.IsNil)
1165
1166         // good AuthSecret
1167         conn, err = s.localdb.ContainerGatewayTunnel(s.userctx, arvados.ContainerGatewayTunnelOptions{
1168                 UUID:       s.ctrUUID,
1169                 AuthSecret: s.gw.AuthSecret,
1170         })
1171         c.Check(err, check.IsNil)
1172         c.Check(conn.Conn, check.NotNil)
1173 }
1174
1175 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyOK(c *check.C) {
1176         forceProxyForTest = true
1177         s.testConnectThroughTunnel(c, "")
1178 }
1179
1180 func (s *ContainerGatewaySuite) TestConnectThroughTunnelWithProxyError(c *check.C) {
1181         forceProxyForTest = true
1182         delete(s.cluster.Services.Controller.InternalURLs, *forceInternalURLForTest)
1183         s.testConnectThroughTunnel(c, `.*tunnel endpoint is invalid.*`)
1184 }
1185
1186 func (s *ContainerGatewaySuite) TestConnectThroughTunnelNoProxyOK(c *check.C) {
1187         s.testConnectThroughTunnel(c, "")
1188 }
1189
1190 func (s *ContainerGatewaySuite) setupGatewayWithTunnel(c *check.C) *crunchrun.Gateway {
1191         rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
1192         // Until the tunnel starts up, set gateway_address to a value
1193         // that can't work. We want to ensure the only way we can
1194         // reach the gateway is through the tunnel.
1195         tungw := &crunchrun.Gateway{
1196                 ContainerUUID: s.ctrUUID,
1197                 AuthSecret:    s.gw.AuthSecret,
1198                 Log:           ctxlog.TestLogger(c),
1199                 Target:        crunchrun.GatewayTargetStub{},
1200                 ArvadosClient: s.gw.ArvadosClient,
1201                 UpdateTunnelURL: func(url string) {
1202                         c.Logf("UpdateTunnelURL(%q)", url)
1203                         gwaddr := "tunnel " + url
1204                         s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
1205                                 UUID: s.ctrUUID,
1206                                 Attrs: map[string]interface{}{
1207                                         "gateway_address": gwaddr}})
1208                 },
1209         }
1210         c.Assert(tungw.Start(), check.IsNil)
1211
1212         // We didn't supply an external hostname in the Address field,
1213         // so Start() should assign a local address.
1214         host, _, err := net.SplitHostPort(tungw.Address)
1215         c.Assert(err, check.IsNil)
1216         c.Check(host, check.Equals, "127.0.0.1")
1217
1218         _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
1219                 UUID: s.ctrUUID,
1220                 Attrs: map[string]interface{}{
1221                         "state": arvados.ContainerStateRunning,
1222                 }})
1223         c.Assert(err, check.IsNil)
1224
1225         for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Second / 2) {
1226                 ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
1227                 c.Assert(err, check.IsNil)
1228                 c.Check(ctr.InteractiveSessionStarted, check.Equals, false)
1229                 c.Logf("ctr.GatewayAddress == %s", ctr.GatewayAddress)
1230                 if strings.HasPrefix(ctr.GatewayAddress, "tunnel ") {
1231                         break
1232                 }
1233         }
1234         return tungw
1235 }
1236
1237 func (s *ContainerGatewaySuite) testConnectThroughTunnel(c *check.C, expectErrorMatch string) {
1238         s.setupGatewayWithTunnel(c)
1239         c.Log("connecting to gateway through tunnel")
1240         arpc := rpc.NewConn("", &url.URL{Scheme: "https", Host: s.gw.ArvadosClient.APIHost}, true, rpc.PassthroughTokenProvider)
1241         sshconn, err := arpc.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})
1242         if expectErrorMatch != "" {
1243                 c.Check(err, check.ErrorMatches, expectErrorMatch)
1244                 return
1245         }
1246         c.Assert(err, check.IsNil)
1247         c.Assert(sshconn.Conn, check.NotNil)
1248         defer sshconn.Conn.Close()
1249
1250         done := make(chan struct{})
1251         go func() {
1252                 defer close(done)
1253
1254                 // Receive text banner
1255                 buf := make([]byte, 12)
1256                 _, err := io.ReadFull(sshconn.Conn, buf)
1257                 c.Check(err, check.IsNil)
1258                 c.Check(string(buf), check.Equals, "SSH-2.0-Go\r\n")
1259
1260                 // Send text banner
1261                 _, err = sshconn.Conn.Write([]byte("SSH-2.0-Fake\r\n"))
1262                 c.Check(err, check.IsNil)
1263
1264                 // Receive binary
1265                 _, err = io.ReadFull(sshconn.Conn, buf[:4])
1266                 c.Check(err, check.IsNil)
1267
1268                 // If we can get this far into an SSH handshake...
1269                 c.Logf("was able to read %x -- success, tunnel is working", buf[:4])
1270         }()
1271         select {
1272         case <-done:
1273         case <-time.After(time.Second):
1274                 c.Fail()
1275         }
1276         ctr, err := s.localdb.ContainerGet(s.userctx, arvados.GetOptions{UUID: s.ctrUUID})
1277         c.Check(err, check.IsNil)
1278         c.Check(ctr.InteractiveSessionStarted, check.Equals, true)
1279 }
1280
1281 func getCookie(resp *http.Response, name string) *http.Cookie {
1282         for _, cookie := range resp.Cookies() {
1283                 if cookie.Name == name {
1284                         return cookie
1285                 }
1286         }
1287         return nil
1288 }