From 8fd331405028ebdbb97de58560057564aa530105 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Tue, 18 May 2021 16:39:00 -0400 Subject: [PATCH] 17296: Add crunch-run integration test. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- lib/crunchrun/crunchrun.go | 19 ++- lib/crunchrun/integration_test.go | 212 ++++++++++++++++++++++++++++++ lib/crunchrun/singularity.go | 10 +- 3 files changed, 234 insertions(+), 7 deletions(-) create mode 100644 lib/crunchrun/integration_test.go diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go index 4bdaca4d64..77cfcf68b7 100644 --- a/lib/crunchrun/crunchrun.go +++ b/lib/crunchrun/crunchrun.go @@ -148,9 +148,10 @@ type ContainerRunner struct { cStateLock sync.Mutex cCancelled bool // StopContainer() invoked - enableNetwork string // one of "default" or "always" - networkMode string // "none", "host", or "" -- passed through to executor - arvMountLog *ThrottledLogger + enableMemoryLimit bool + enableNetwork string // one of "default" or "always" + networkMode string // "none", "host", or "" -- passed through to executor + arvMountLog *ThrottledLogger containerWatchdogInterval time.Duration @@ -291,7 +292,7 @@ func (runner *ContainerRunner) ArvMountCmd(arvMountCmd []string, token string) ( } runner.arvMountLog = NewThrottledLogger(w) c.Stdout = runner.arvMountLog - c.Stderr = runner.arvMountLog + c.Stderr = io.MultiWriter(runner.arvMountLog, os.Stderr) runner.CrunchLog.Printf("Running %v", c.Args) @@ -386,6 +387,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) { if err != nil { return nil, fmt.Errorf("could not get container token: %s", err) } + runner.CrunchLog.Printf("container token %q", token) pdhOnly := true tmpcount := 0 @@ -946,11 +948,14 @@ func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[st // both "" and "." mean default workdir = "" } - + ram := runner.Container.RuntimeConstraints.RAM + if !runner.enableMemoryLimit { + ram = 0 + } return runner.executor.Create(containerSpec{ Image: imageID, VCPUs: runner.Container.RuntimeConstraints.VCPUs, - RAM: runner.Container.RuntimeConstraints.RAM, + RAM: ram, WorkingDir: workdir, Env: env, BindMounts: bindmounts, @@ -1586,6 +1591,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s sleep := flags.Duration("sleep", 0, "Delay before starting (testing use only)") kill := flags.Int("kill", -1, "Send signal to an existing crunch-run process for given UUID") list := flags.Bool("list", false, "List UUIDs of existing crunch-run processes") + enableMemoryLimit := flags.Bool("enable-memory-limit", true, "tell container runtime to limit container's memory usage") enableNetwork := flags.String("container-enable-networking", "default", "enable networking \"always\" (for all containers) or \"default\" (for containers that request it)") networkMode := flags.String("container-network-mode", "default", `Docker network mode for container (use any argument valid for docker --net)`) memprofile := flags.String("memprofile", "", "write memory profile to `file` after running container") @@ -1718,6 +1724,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s cr.statInterval = *statInterval cr.cgroupRoot = *cgroupRoot cr.expectCgroupParent = *cgroupParent + cr.enableMemoryLimit = *enableMemoryLimit cr.enableNetwork = *enableNetwork cr.networkMode = *networkMode if *cgroupParentSubsystem != "" { diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go new file mode 100644 index 0000000000..04a15bcea7 --- /dev/null +++ b/lib/crunchrun/integration_test.go @@ -0,0 +1,212 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package crunchrun + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "strings" + + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/keepclient" + . "gopkg.in/check.v1" +) + +var _ = Suite(&integrationSuite{}) + +type integrationSuite struct { + engine string + image arvados.Collection + input arvados.Collection + stdin bytes.Buffer + stdout bytes.Buffer + stderr bytes.Buffer + cr arvados.ContainerRequest + client *arvados.Client + ac *arvadosclient.ArvadosClient + kc *keepclient.KeepClient +} + +func (s *integrationSuite) SetUpSuite(c *C) { + arvadostest.StartKeep(2, true) + + out, err := exec.Command("docker", "load", "--input", busyboxDockerImage(c)).CombinedOutput() + c.Log(string(out)) + c.Assert(err, IsNil) + out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output() + imageUUID := strings.TrimSpace(string(out)) + c.Logf("image uuid %s", imageUUID) + c.Assert(err, IsNil) + err = arvados.NewClientFromEnv().RequestAndDecode(&s.image, "GET", "arvados/v1/collections/"+imageUUID, nil, nil) + c.Assert(err, IsNil) + c.Logf("image pdh %s", s.image.PortableDataHash) + + s.client = arvados.NewClientFromEnv() + s.ac, err = arvadosclient.New(s.client) + c.Assert(err, IsNil) + s.kc = keepclient.New(s.ac) + fs, err := s.input.FileSystem(s.client, s.kc) + c.Assert(err, IsNil) + f, err := fs.OpenFile("inputfile", os.O_CREATE|os.O_WRONLY, 0755) + c.Assert(err, IsNil) + _, err = f.Write([]byte("inputdata")) + c.Assert(err, IsNil) + err = f.Close() + c.Assert(err, IsNil) + s.input.ManifestText, err = fs.MarshalManifest(".") + c.Assert(err, IsNil) + err = s.client.RequestAndDecode(&s.input, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "ensure_unique_name": true, + "collection": map[string]interface{}{ + "manifest_text": s.input.ManifestText, + }, + }) + c.Assert(err, IsNil) + c.Logf("input pdh %s", s.input.PortableDataHash) +} + +func (s *integrationSuite) TearDownSuite(c *C) { + err := s.client.RequestAndDecode(nil, "POST", "database/reset", nil, nil) + c.Check(err, IsNil) +} + +func (s *integrationSuite) SetUpTest(c *C) { + s.engine = "docker" + s.stdin = bytes.Buffer{} + s.stdout = bytes.Buffer{} + s.stderr = bytes.Buffer{} + s.cr = arvados.ContainerRequest{ + Priority: 1, + State: "Committed", + OutputPath: "/mnt/out", + ContainerImage: s.image.PortableDataHash, + Mounts: map[string]arvados.Mount{ + "/mnt/json": { + Kind: "json", + Content: []interface{}{ + "foo", + map[string]string{"foo": "bar"}, + nil, + }, + }, + "/mnt/in": { + Kind: "collection", + PortableDataHash: s.input.PortableDataHash, + }, + "/mnt/out": { + Kind: "tmp", + Capacity: 1000, + }, + }, + RuntimeConstraints: arvados.RuntimeConstraints{ + RAM: 128000000, + VCPUs: 1, + API: true, + }, + } +} + +func (s *integrationSuite) setup(c *C) { + err := s.client.RequestAndDecode(&s.cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{ + "priority": s.cr.Priority, + "state": s.cr.State, + "command": s.cr.Command, + "output_path": s.cr.OutputPath, + "container_image": s.cr.ContainerImage, + "mounts": s.cr.Mounts, + "runtime_constraints": s.cr.RuntimeConstraints, + "use_existing": false, + }}) + c.Assert(err, IsNil) + c.Assert(s.cr.ContainerUUID, Not(Equals), "") + err = s.client.RequestAndDecode(nil, "POST", "arvados/v1/containers/"+s.cr.ContainerUUID+"/lock", nil, nil) + c.Assert(err, IsNil) +} + +func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) { + s.engine = "docker" + s.testRunTrivialContainer(c) +} + +func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) { + s.engine = "singularity" + s.testRunTrivialContainer(c) +} + +func (s *integrationSuite) testRunTrivialContainer(c *C) { + if err := exec.Command("which", s.engine).Run(); err != nil { + c.Skip(fmt.Sprintf("%s: %s", s.engine, err)) + } + s.cr.Command = []string{"sh", "-c", "cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"} + s.setup(c) + code := command{}.RunCommand("crunch-run", []string{ + "-runtime-engine=" + s.engine, + "-enable-memory-limit=false", + s.cr.ContainerUUID, + }, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr)) + c.Check(code, Equals, 0) + err := s.client.RequestAndDecode(&s.cr, "GET", "arvados/v1/container_requests/"+s.cr.UUID, nil, nil) + c.Assert(err, IsNil) + c.Logf("Finished container request: %#v", s.cr) + + var log arvados.Collection + err = s.client.RequestAndDecode(&log, "GET", "arvados/v1/collections/"+s.cr.LogUUID, nil, nil) + c.Assert(err, IsNil) + fs, err := log.FileSystem(s.client, s.kc) + c.Assert(err, IsNil) + if d, err := fs.Open("/"); c.Check(err, IsNil) { + fis, err := d.Readdir(-1) + c.Assert(err, IsNil) + for _, fi := range fis { + if fi.IsDir() { + continue + } + f, err := fs.Open(fi.Name()) + c.Assert(err, IsNil) + buf, err := ioutil.ReadAll(f) + c.Assert(err, IsNil) + c.Logf("\n===== %s =====\n%s", fi.Name(), buf) + } + } + + var output arvados.Collection + err = s.client.RequestAndDecode(&output, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil) + c.Assert(err, IsNil) + fs, err = output.FileSystem(s.client, s.kc) + c.Assert(err, IsNil) + if f, err := fs.Open("inputfile"); c.Check(err, IsNil) { + defer f.Close() + buf, err := ioutil.ReadAll(f) + c.Check(err, IsNil) + c.Check(string(buf), Equals, "inputdata") + } + if f, err := fs.Open("json"); c.Check(err, IsNil) { + defer f.Close() + buf, err := ioutil.ReadAll(f) + c.Check(err, IsNil) + c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`) + } + if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) { + c.Check(fi.IsDir(), Equals, true) + } + if d, err := fs.Open("emptydir"); c.Check(err, IsNil) { + defer d.Close() + fis, err := d.Readdir(-1) + c.Assert(err, IsNil) + // crunch-run still saves a ".keep" file to preserve + // empty dirs even though that shouldn't be + // necessary. Ideally we would do: + // c.Check(fis, HasLen, 0) + for _, fi := range fis { + c.Check(fi.Name(), Equals, ".keep") + } + } +} diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go index d783baab9f..4bec8c3ebe 100644 --- a/lib/crunchrun/singularity.go +++ b/lib/crunchrun/singularity.go @@ -40,8 +40,16 @@ func (e *singularityExecutor) ImageLoaded(string) bool { // containerImage into a sif file for later use. func (e *singularityExecutor) LoadImage(imageTarballPath string) error { e.logf("building singularity image") + // "singularity build" does not accept a + // docker-archive://... filename containing a ":" character, + // as in "/path/to/sha256:abcd...1234.tar". Workaround: make a + // symlink that doesn't have ":" chars. + err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar") + if err != nil { + return err + } e.imageFilename = e.tmpdir + "/image.sif" - build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+imageTarballPath) + build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+e.tmpdir+"/image.tar") e.logf("%v", build.Args) out, err := build.CombinedOutput() // INFO: Starting build... -- 2.30.2