X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/64c516079154f73da3f2a33a957fa8ae8eb23749..2e74236fa27822addd856f194befc28382990ce0:/lib/crunchrun/integration_test.go diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go index c688248c64..a9a270b07b 100644 --- a/lib/crunchrun/integration_test.go +++ b/lib/crunchrun/integration_test.go @@ -6,6 +6,7 @@ package crunchrun import ( "bytes" + "encoding/json" "fmt" "io" "io/ioutil" @@ -13,9 +14,11 @@ import ( "os/exec" "strings" + "git.arvados.org/arvados.git/lib/config" "git.arvados.org/arvados.git/sdk/go/arvados" "git.arvados.org/arvados.git/sdk/go/arvadosclient" "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" "git.arvados.org/arvados.git/sdk/go/keepclient" . "gopkg.in/check.v1" ) @@ -29,10 +32,15 @@ type integrationSuite struct { stdin bytes.Buffer stdout bytes.Buffer stderr bytes.Buffer + args []string cr arvados.ContainerRequest client *arvados.Client ac *arvadosclient.ArvadosClient kc *keepclient.KeepClient + + logCollection arvados.Collection + outputCollection arvados.Collection + logFiles map[string]string // filename => contents } func (s *integrationSuite) SetUpSuite(c *C) { @@ -43,13 +51,18 @@ func (s *integrationSuite) SetUpSuite(c *C) { arvadostest.StartKeep(2, true) - out, err := exec.Command("docker", "load", "--input", busyboxDockerImage(c)).CombinedOutput() + out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput() c.Log(string(out)) c.Assert(err, IsNil) out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output() imageUUID := strings.TrimSpace(string(out)) c.Logf("image uuid %s", imageUUID) - c.Assert(err, IsNil) + if !c.Check(err, IsNil) { + if err, ok := err.(*exec.ExitError); ok { + c.Logf("%s", err.Stderr) + } + c.Fail() + } err = arvados.NewClientFromEnv().RequestAndDecode(&s.image, "GET", "arvados/v1/collections/"+imageUUID, nil, nil) c.Assert(err, IsNil) c.Logf("image pdh %s", s.image.PortableDataHash) @@ -79,6 +92,7 @@ func (s *integrationSuite) SetUpSuite(c *C) { } func (s *integrationSuite) TearDownSuite(c *C) { + os.Unsetenv("ARVADOS_KEEP_SERVICES") if s.client == nil { // didn't set up return @@ -88,10 +102,15 @@ func (s *integrationSuite) TearDownSuite(c *C) { } func (s *integrationSuite) SetUpTest(c *C) { + os.Unsetenv("ARVADOS_KEEP_SERVICES") s.engine = "docker" + s.args = nil s.stdin = bytes.Buffer{} s.stdout = bytes.Buffer{} s.stderr = bytes.Buffer{} + s.logCollection = arvados.Collection{} + s.outputCollection = arvados.Collection{} + s.logFiles = map[string]string{} s.cr = arvados.ContainerRequest{ Priority: 1, State: "Committed", @@ -143,11 +162,87 @@ func (s *integrationSuite) setup(c *C) { func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) { s.engine = "docker" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: docker Engine \d+\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) { s.engine = "singularity" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: singularity.* version 3\.\d+.*`) +} + +func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) { + for _, trial := range []struct { + logConfig string + matchGetReq Checker + matchPutReq Checker + matchStartupMessage Checker + }{ + {"none", Not(Matches), Not(Matches), Not(Matches)}, + {"all", Matches, Matches, Matches}, + {"errors", Not(Matches), Not(Matches), Matches}, + } { + c.Logf("=== testing with Containers.LocalKeepLogsToContainerLog: %q", trial.logConfig) + s.SetUpTest(c) + + cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load() + c.Assert(err, IsNil) + cluster, err := cfg.GetCluster("") + c.Assert(err, IsNil) + for uuid, volume := range cluster.Volumes { + volume.AccessViaHosts = nil + volume.Replication = 2 + cluster.Volumes[uuid] = volume + } + cluster.Containers.LocalKeepLogsToContainerLog = trial.logConfig + + s.stdin.Reset() + err = json.NewEncoder(&s.stdin).Encode(ConfigData{ + Env: nil, + KeepBuffers: 1, + Cluster: cluster, + }) + c.Assert(err, IsNil) + + s.engine = "docker" + s.testRunTrivialContainer(c) + + log, logExists := s.logFiles["keepstore.txt"] + if trial.logConfig == "none" { + c.Check(logExists, Equals, false) + } else { + c.Check(log, trial.matchGetReq, `(?ms).*"reqMethod":"GET".*`) + c.Check(log, trial.matchPutReq, `(?ms).*"reqMethod":"PUT".*,"reqPath":"0e3bcff26d51c895a60ea0d4585e134d".*`) + } + } + + // Check that (1) config is loaded from $ARVADOS_CONFIG when + // not provided on stdin and (2) if a local keepstore is not + // started, crunch-run.txt explains why not. + s.SetUpTest(c) + s.stdin.Reset() + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because a volume \(zzzzz-nyw5e-00000000000000\d\) uses AccessViaHosts\n.*`) + + // Check that config read errors are logged + s.SetUpTest(c) + s.args = []string{"-config", c.MkDir() + "/config-error.yaml"} + s.stdin.Reset() + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* no such file or directory\n.*`) + + s.SetUpTest(c) + s.args = []string{"-config", c.MkDir() + "/config-unreadable.yaml"} + s.stdin.Reset() + err := ioutil.WriteFile(s.args[1], []byte{}, 0) + c.Check(err, IsNil) + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* permission denied\n.*`) + + s.SetUpTest(c) + s.stdin.Reset() + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*loaded config file \Q`+os.Getenv("ARVADOS_CONFIG")+`\E\n.*`) } func (s *integrationSuite) testRunTrivialContainer(c *C) { @@ -156,11 +251,19 @@ func (s *integrationSuite) testRunTrivialContainer(c *C) { } s.cr.Command = []string{"sh", "-c", "cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"} s.setup(c) - code := command{}.RunCommand("crunch-run", []string{ + + args := []string{ "-runtime-engine=" + s.engine, "-enable-memory-limit=false", - s.cr.ContainerUUID, - }, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr)) + } + if s.stdin.Len() > 0 { + args = append(args, "-stdin-config=true") + } + args = append(args, s.args...) + args = append(args, s.cr.ContainerUUID) + code := command{}.RunCommand("crunch-run", args, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr)) + c.Logf("\n===== stdout =====\n%s", s.stdout.String()) + c.Logf("\n===== stderr =====\n%s", s.stderr.String()) c.Check(code, Equals, 0) err := s.client.RequestAndDecode(&s.cr, "GET", "arvados/v1/container_requests/"+s.cr.UUID, nil, nil) c.Assert(err, IsNil) @@ -183,8 +286,10 @@ func (s *integrationSuite) testRunTrivialContainer(c *C) { buf, err := ioutil.ReadAll(f) c.Assert(err, IsNil) c.Logf("\n===== %s =====\n%s", fi.Name(), buf) + s.logFiles[fi.Name()] = string(buf) } } + s.logCollection = log var output arvados.Collection err = s.client.RequestAndDecode(&output, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil) @@ -218,4 +323,5 @@ func (s *integrationSuite) testRunTrivialContainer(c *C) { c.Check(fi.Name(), Equals, ".keep") } } + s.outputCollection = output }