package localdb
import (
+ "bytes"
"crypto/hmac"
"crypto/sha256"
"fmt"
"net/http/httptest"
"net/url"
"os"
+ "os/exec"
+ "path/filepath"
"strings"
"time"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"golang.org/x/crypto/ssh"
check "gopkg.in/check.v1"
type ContainerGatewaySuite struct {
localdbSuite
ctrUUID string
+ srv *httptest.Server
gw *crunchrun.Gateway
}
authKey := fmt.Sprintf("%x", h.Sum(nil))
rtr := router.New(s.localdb, router.Config{})
- srv := httptest.NewUnstartedServer(rtr)
- srv.StartTLS()
+ s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
+ s.srv.StartTLS()
// the test setup doesn't use lib/service so
// service.URLFromContext() returns nothing -- instead, this
// is how we advertise our internal URL and enable
// proxy-to-other-controller mode,
- forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: srv.Listener.Addr().String()}
+ forceInternalURLForTest = &arvados.URL{Scheme: "https", Host: s.srv.Listener.Addr().String()}
ac := &arvados.Client{
- APIHost: srv.Listener.Addr().String(),
+ APIHost: s.srv.Listener.Addr().String(),
AuthToken: arvadostest.Dispatch1Token,
Insecure: true,
}
c.Check(err, check.IsNil)
}
+func (s *ContainerGatewaySuite) TearDownTest(c *check.C) {
+ s.srv.Close()
+ s.localdbSuite.TearDownTest(c)
+}
+
func (s *ContainerGatewaySuite) TestConfig(c *check.C) {
for _, trial := range []struct {
configAdmin bool
}
}
-func (s *ContainerGatewaySuite) setupLogCollection(c *check.C, files map[string]string) {
+func (s *ContainerGatewaySuite) setupLogCollection(c *check.C) {
+ files := map[string]string{
+ "stderr.txt": "hello world\n",
+ "a/b/c/d.html": "<html></html>\n",
+ }
client := arvados.NewClientFromEnv()
ac, err := arvadosclient.New(client)
c.Assert(err, check.IsNil)
s.gw.LogCollection = cfs
}
+func (s *ContainerGatewaySuite) saveLogAndCloseGateway(c *check.C) {
+ rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
+ txt, err := s.gw.LogCollection.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
+ Attrs: map[string]interface{}{
+ "manifest_text": txt,
+ }})
+ c.Assert(err, check.IsNil)
+ _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ UUID: s.ctrUUID,
+ Attrs: map[string]interface{}{
+ "log": coll.PortableDataHash,
+ "gateway_address": "",
+ }})
+ c.Assert(err, check.IsNil)
+ // gateway_address="" above already ensures localdb
+ // can't circumvent the keep-web proxy test by getting
+ // content from the container gateway; this is just
+ // extra insurance.
+ s.gw.LogCollection = nil
+}
+
func (s *ContainerGatewaySuite) TestContainerLogViaTunnel(c *check.C) {
forceProxyForTest = true
defer func() { forceProxyForTest = false }()
s.gw = s.setupGatewayWithTunnel(c)
- s.setupLogCollection(c, map[string]string{
- "stderr.txt": "hello world\n",
- })
+ s.setupLogCollection(c)
for _, broken := range []bool{false, true} {
c.Logf("broken=%v", broken)
}
func (s *ContainerGatewaySuite) TestContainerLogViaGateway(c *check.C) {
- s.testContainerLog(c, true)
+ s.setupLogCollection(c)
+ s.testContainerLog(c)
}
func (s *ContainerGatewaySuite) TestContainerLogViaKeepWeb(c *check.C) {
- s.testContainerLog(c, false)
+ s.setupLogCollection(c)
+ s.saveLogAndCloseGateway(c)
+ s.testContainerLog(c)
}
-func (s *ContainerGatewaySuite) testContainerLog(c *check.C, viaGateway bool) {
- s.setupLogCollection(c, map[string]string{
- "stderr.txt": "hello world\n",
- "a/b/c/d.html": "<html></html>\n",
- })
- if !viaGateway {
- rootctx := ctrlctx.NewWithToken(s.ctx, s.cluster, s.cluster.SystemRootToken)
- txt, err := s.gw.LogCollection.MarshalManifest(".")
- c.Assert(err, check.IsNil)
- coll, err := s.localdb.CollectionCreate(rootctx, arvados.CreateOptions{
- Attrs: map[string]interface{}{
- "manifest_text": txt,
- }})
- c.Assert(err, check.IsNil)
- _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
- UUID: s.ctrUUID,
- Attrs: map[string]interface{}{
- "log": coll.PortableDataHash,
- "gateway_address": "",
- }})
- c.Assert(err, check.IsNil)
- // gateway_address="" above already ensures localdb
- // can't circumvent the keep-web proxy test by getting
- // content from the container gateway; this is just
- // extra insurance.
- s.gw.LogCollection = nil
- }
+func (s *ContainerGatewaySuite) testContainerLog(c *check.C) {
for _, trial := range []struct {
method string
path string
}
}
+func (s *ContainerGatewaySuite) TestContainerLogViaCadaver(c *check.C) {
+ s.setupLogCollection(c)
+
+ out := s.runCadaver(c, arvadostest.ActiveToken, "/arvados/v1/containers/"+s.ctrUUID+"/log", "ls")
+ c.Check(out, check.Matches, `(?ms).*stderr\.txt\s+12\s.*`)
+ c.Check(out, check.Matches, `(?ms).*a\s+0\s.*`)
+
+ out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/containers/"+s.ctrUUID+"/log", "get stderr.txt")
+ c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
+
+ s.saveLogAndCloseGateway(c)
+
+ out = s.runCadaver(c, arvadostest.ActiveTokenV2, "/arvados/v1/containers/"+s.ctrUUID+"/log", "get stderr.txt")
+ c.Check(out, check.Matches, `(?ms).*Downloading .* to stderr\.txt: .* succeeded\..*`)
+}
+
+func (s *ContainerGatewaySuite) runCadaver(c *check.C, password, path, stdin string) string {
+ // Replace s.srv with an HTTP server, otherwise cadaver will
+ // just fail on TLS cert verification.
+ s.srv.Close()
+ rtr := router.New(s.localdb, router.Config{})
+ s.srv = httptest.NewUnstartedServer(httpserver.AddRequestIDs(httpserver.LogRequests(rtr)))
+ s.srv.Start()
+
+ tempdir, err := ioutil.TempDir("", "localdb-test-")
+ c.Assert(err, check.IsNil)
+ defer os.RemoveAll(tempdir)
+
+ cmd := exec.Command("cadaver", s.srv.URL+path)
+ if password != "" {
+ cmd.Env = append(os.Environ(), "HOME="+tempdir)
+ f, err := os.OpenFile(filepath.Join(tempdir, ".netrc"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+ c.Assert(err, check.IsNil)
+ _, err = fmt.Fprintf(f, "default login none password %s\n", password)
+ c.Assert(err, check.IsNil)
+ c.Assert(f.Close(), check.IsNil)
+ }
+ cmd.Stdin = bytes.NewBufferString(stdin)
+ cmd.Dir = tempdir
+ stdout, err := cmd.StdoutPipe()
+ c.Assert(err, check.Equals, nil)
+ cmd.Stderr = cmd.Stdout
+ c.Logf("cmd: %v", cmd.Args)
+ go cmd.Start()
+
+ var buf bytes.Buffer
+ _, err = io.Copy(&buf, stdout)
+ c.Check(err, check.Equals, nil)
+ err = cmd.Wait()
+ c.Check(err, check.Equals, nil)
+ return buf.String()
+}
+
func (s *ContainerGatewaySuite) TestConnect(c *check.C) {
c.Logf("connecting to %s", s.gw.Address)
sshconn, err := s.localdb.ContainerSSH(s.userctx, arvados.ContainerSSHOptions{UUID: s.ctrUUID})