17296: Add crunch-run integration test.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 4bdaca4d64b7c74d36fab5b17443b3f61bdb334e..77cfcf68b7342a6b28fc912efdb577f59d687cdf 100644 (file)
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -148,9 +148,10 @@ type ContainerRunner struct {
        cStateLock sync.Mutex
        cCancelled bool // StopContainer() invoked
 
-       enableNetwork string // one of "default" or "always"
-       networkMode   string // "none", "host", or "" -- passed through to executor
-       arvMountLog   *ThrottledLogger
+       enableMemoryLimit bool
+       enableNetwork     string // one of "default" or "always"
+       networkMode       string // "none", "host", or "" -- passed through to executor
+       arvMountLog       *ThrottledLogger
 
        containerWatchdogInterval time.Duration
 
@@ -291,7 +292,7 @@ func (runner *ContainerRunner) ArvMountCmd(arvMountCmd []string, token string) (
        }
        runner.arvMountLog = NewThrottledLogger(w)
        c.Stdout = runner.arvMountLog
-       c.Stderr = runner.arvMountLog
+       c.Stderr = io.MultiWriter(runner.arvMountLog, os.Stderr)
 
        runner.CrunchLog.Printf("Running %v", c.Args)
 
@@ -386,6 +387,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
        if err != nil {
                return nil, fmt.Errorf("could not get container token: %s", err)
        }
+       runner.CrunchLog.Printf("container token %q", token)
 
        pdhOnly := true
        tmpcount := 0
@@ -946,11 +948,14 @@ func (runner *ContainerRunner) CreateContainer(imageID string, bindmounts map[st
                // both "" and "." mean default
                workdir = ""
        }
-
+       ram := runner.Container.RuntimeConstraints.RAM
+       if !runner.enableMemoryLimit {
+               ram = 0
+       }
        return runner.executor.Create(containerSpec{
                Image:         imageID,
                VCPUs:         runner.Container.RuntimeConstraints.VCPUs,
-               RAM:           runner.Container.RuntimeConstraints.RAM,
+               RAM:           ram,
                WorkingDir:    workdir,
                Env:           env,
                BindMounts:    bindmounts,
@@ -1586,6 +1591,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
        sleep := flags.Duration("sleep", 0, "Delay before starting (testing use only)")
        kill := flags.Int("kill", -1, "Send signal to an existing crunch-run process for given UUID")
        list := flags.Bool("list", false, "List UUIDs of existing crunch-run processes")
+       enableMemoryLimit := flags.Bool("enable-memory-limit", true, "tell container runtime to limit container's memory usage")
        enableNetwork := flags.String("container-enable-networking", "default", "enable networking \"always\" (for all containers) or \"default\" (for containers that request it)")
        networkMode := flags.String("container-network-mode", "default", `Docker network mode for container (use any argument valid for docker --net)`)
        memprofile := flags.String("memprofile", "", "write memory profile to `file` after running container")
@@ -1718,6 +1724,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
        cr.statInterval = *statInterval
        cr.cgroupRoot = *cgroupRoot
        cr.expectCgroupParent = *cgroupParent
+       cr.enableMemoryLimit = *enableMemoryLimit
        cr.enableNetwork = *enableNetwork
        cr.networkMode = *networkMode
        if *cgroupParentSubsystem != "" {

diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go
new file mode 100644 (file)
index 0000000..04a15bc
--- /dev/null
+++ b/lib/crunchrun/integration_test.go
@@ -0,0 +1,212 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package crunchrun
+
+import (
+       "bytes"
+       "fmt"
+       "io"
+       "io/ioutil"
+       "os"
+       "os/exec"
+       "strings"
+
+       "git.arvados.org/arvados.git/sdk/go/arvados"
+       "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "git.arvados.org/arvados.git/sdk/go/keepclient"
+       . "gopkg.in/check.v1"
+)
+
+var _ = Suite(&integrationSuite{})
+
+type integrationSuite struct {
+       engine string
+       image  arvados.Collection
+       input  arvados.Collection
+       stdin  bytes.Buffer
+       stdout bytes.Buffer
+       stderr bytes.Buffer
+       cr     arvados.ContainerRequest
+       client *arvados.Client
+       ac     *arvadosclient.ArvadosClient
+       kc     *keepclient.KeepClient
+}
+
+func (s *integrationSuite) SetUpSuite(c *C) {
+       arvadostest.StartKeep(2, true)
+
+       out, err := exec.Command("docker", "load", "--input", busyboxDockerImage(c)).CombinedOutput()
+       c.Log(string(out))
+       c.Assert(err, IsNil)
+       out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output()
+       imageUUID := strings.TrimSpace(string(out))
+       c.Logf("image uuid %s", imageUUID)
+       c.Assert(err, IsNil)
+       err = arvados.NewClientFromEnv().RequestAndDecode(&s.image, "GET", "arvados/v1/collections/"+imageUUID, nil, nil)
+       c.Assert(err, IsNil)
+       c.Logf("image pdh %s", s.image.PortableDataHash)
+
+       s.client = arvados.NewClientFromEnv()
+       s.ac, err = arvadosclient.New(s.client)
+       c.Assert(err, IsNil)
+       s.kc = keepclient.New(s.ac)
+       fs, err := s.input.FileSystem(s.client, s.kc)
+       c.Assert(err, IsNil)
+       f, err := fs.OpenFile("inputfile", os.O_CREATE|os.O_WRONLY, 0755)
+       c.Assert(err, IsNil)
+       _, err = f.Write([]byte("inputdata"))
+       c.Assert(err, IsNil)
+       err = f.Close()
+       c.Assert(err, IsNil)
+       s.input.ManifestText, err = fs.MarshalManifest(".")
+       c.Assert(err, IsNil)
+       err = s.client.RequestAndDecode(&s.input, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+               "ensure_unique_name": true,
+               "collection": map[string]interface{}{
+                       "manifest_text": s.input.ManifestText,
+               },
+       })
+       c.Assert(err, IsNil)
+       c.Logf("input pdh %s", s.input.PortableDataHash)
+}
+
+func (s *integrationSuite) TearDownSuite(c *C) {
+       err := s.client.RequestAndDecode(nil, "POST", "database/reset", nil, nil)
+       c.Check(err, IsNil)
+}
+
+func (s *integrationSuite) SetUpTest(c *C) {
+       s.engine = "docker"
+       s.stdin = bytes.Buffer{}
+       s.stdout = bytes.Buffer{}
+       s.stderr = bytes.Buffer{}
+       s.cr = arvados.ContainerRequest{
+               Priority:       1,
+               State:          "Committed",
+               OutputPath:     "/mnt/out",
+               ContainerImage: s.image.PortableDataHash,
+               Mounts: map[string]arvados.Mount{
+                       "/mnt/json": {
+                               Kind: "json",
+                               Content: []interface{}{
+                                       "foo",
+                                       map[string]string{"foo": "bar"},
+                                       nil,
+                               },
+                       },
+                       "/mnt/in": {
+                               Kind:             "collection",
+                               PortableDataHash: s.input.PortableDataHash,
+                       },
+                       "/mnt/out": {
+                               Kind:     "tmp",
+                               Capacity: 1000,
+                       },
+               },
+               RuntimeConstraints: arvados.RuntimeConstraints{
+                       RAM:   128000000,
+                       VCPUs: 1,
+                       API:   true,
+               },
+       }
+}
+
+func (s *integrationSuite) setup(c *C) {
+       err := s.client.RequestAndDecode(&s.cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
+               "priority":            s.cr.Priority,
+               "state":               s.cr.State,
+               "command":             s.cr.Command,
+               "output_path":         s.cr.OutputPath,
+               "container_image":     s.cr.ContainerImage,
+               "mounts":              s.cr.Mounts,
+               "runtime_constraints": s.cr.RuntimeConstraints,
+               "use_existing":        false,
+       }})
+       c.Assert(err, IsNil)
+       c.Assert(s.cr.ContainerUUID, Not(Equals), "")
+       err = s.client.RequestAndDecode(nil, "POST", "arvados/v1/containers/"+s.cr.ContainerUUID+"/lock", nil, nil)
+       c.Assert(err, IsNil)
+}
+
+func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) {
+       s.engine = "docker"
+       s.testRunTrivialContainer(c)
+}
+
+func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) {
+       s.engine = "singularity"
+       s.testRunTrivialContainer(c)
+}
+
+func (s *integrationSuite) testRunTrivialContainer(c *C) {
+       if err := exec.Command("which", s.engine).Run(); err != nil {
+               c.Skip(fmt.Sprintf("%s: %s", s.engine, err))
+       }
+       s.cr.Command = []string{"sh", "-c", "cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"}
+       s.setup(c)
+       code := command{}.RunCommand("crunch-run", []string{
+               "-runtime-engine=" + s.engine,
+               "-enable-memory-limit=false",
+               s.cr.ContainerUUID,
+       }, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr))
+       c.Check(code, Equals, 0)
+       err := s.client.RequestAndDecode(&s.cr, "GET", "arvados/v1/container_requests/"+s.cr.UUID, nil, nil)
+       c.Assert(err, IsNil)
+       c.Logf("Finished container request: %#v", s.cr)
+
+       var log arvados.Collection
+       err = s.client.RequestAndDecode(&log, "GET", "arvados/v1/collections/"+s.cr.LogUUID, nil, nil)
+       c.Assert(err, IsNil)
+       fs, err := log.FileSystem(s.client, s.kc)
+       c.Assert(err, IsNil)
+       if d, err := fs.Open("/"); c.Check(err, IsNil) {
+               fis, err := d.Readdir(-1)
+               c.Assert(err, IsNil)
+               for _, fi := range fis {
+                       if fi.IsDir() {
+                               continue
+                       }
+                       f, err := fs.Open(fi.Name())
+                       c.Assert(err, IsNil)
+                       buf, err := ioutil.ReadAll(f)
+                       c.Assert(err, IsNil)
+                       c.Logf("\n===== %s =====\n%s", fi.Name(), buf)
+               }
+       }
+
+       var output arvados.Collection
+       err = s.client.RequestAndDecode(&output, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil)
+       c.Assert(err, IsNil)
+       fs, err = output.FileSystem(s.client, s.kc)
+       c.Assert(err, IsNil)
+       if f, err := fs.Open("inputfile"); c.Check(err, IsNil) {
+               defer f.Close()
+               buf, err := ioutil.ReadAll(f)
+               c.Check(err, IsNil)
+               c.Check(string(buf), Equals, "inputdata")
+       }
+       if f, err := fs.Open("json"); c.Check(err, IsNil) {
+               defer f.Close()
+               buf, err := ioutil.ReadAll(f)
+               c.Check(err, IsNil)
+               c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`)
+       }
+       if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) {
+               c.Check(fi.IsDir(), Equals, true)
+       }
+       if d, err := fs.Open("emptydir"); c.Check(err, IsNil) {
+               defer d.Close()
+               fis, err := d.Readdir(-1)
+               c.Assert(err, IsNil)
+               // crunch-run still saves a ".keep" file to preserve
+               // empty dirs even though that shouldn't be
+               // necessary. Ideally we would do:
+               // c.Check(fis, HasLen, 0)
+               for _, fi := range fis {
+                       c.Check(fi.Name(), Equals, ".keep")
+               }
+       }
+}

diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index d783baab9f9cfab2780387879cd467b81d881e0b..4bec8c3ebed11970c9f0c0734e625c5c32df2523 100644 (file)
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -40,8 +40,16 @@ func (e *singularityExecutor) ImageLoaded(string) bool {
 // containerImage into a sif file for later use.
 func (e *singularityExecutor) LoadImage(imageTarballPath string) error {
        e.logf("building singularity image")
+       // "singularity build" does not accept a
+       // docker-archive://... filename containing a ":" character,
+       // as in "/path/to/sha256:abcd...1234.tar". Workaround: make a
+       // symlink that doesn't have ":" chars.
+       err := os.Symlink(imageTarballPath, e.tmpdir+"/image.tar")
+       if err != nil {
+               return err
+       }
        e.imageFilename = e.tmpdir + "/image.sif"
-       build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+imageTarballPath)
+       build := exec.Command("singularity", "build", e.imageFilename, "docker-archive://"+e.tmpdir+"/image.tar")
        e.logf("%v", build.Args)
        out, err := build.CombinedOutput()
        // INFO:    Starting build...