X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/66a089434f38163273c3a5b9138f9c4347873b69..6de69ae50be42cd1f68d51247b09b995079c0ef4:/lib/crunchrun/integration_test.go diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go index 9b797fd867..38c589f698 100644 --- a/lib/crunchrun/integration_test.go +++ b/lib/crunchrun/integration_test.go @@ -32,6 +32,7 @@ type integrationSuite struct { stdin bytes.Buffer stdout bytes.Buffer stderr bytes.Buffer + args []string cr arvados.ContainerRequest client *arvados.Client ac *arvadosclient.ArvadosClient @@ -39,6 +40,7 @@ type integrationSuite struct { logCollection arvados.Collection outputCollection arvados.Collection + logFiles map[string]string // filename => contents } func (s *integrationSuite) SetUpSuite(c *C) { @@ -49,7 +51,7 @@ func (s *integrationSuite) SetUpSuite(c *C) { arvadostest.StartKeep(2, true) - out, err := exec.Command("docker", "load", "--input", busyboxDockerImage(c)).CombinedOutput() + out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput() c.Log(string(out)) c.Assert(err, IsNil) out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output() @@ -102,11 +104,13 @@ func (s *integrationSuite) TearDownSuite(c *C) { func (s *integrationSuite) SetUpTest(c *C) { os.Unsetenv("ARVADOS_KEEP_SERVICES") s.engine = "docker" + s.args = nil s.stdin = bytes.Buffer{} s.stdout = bytes.Buffer{} s.stderr = bytes.Buffer{} s.logCollection = arvados.Collection{} s.outputCollection = arvados.Collection{} + s.logFiles = map[string]string{} s.cr = arvados.ContainerRequest{ Priority: 1, State: "Committed", @@ -144,6 +148,7 @@ func (s *integrationSuite) setup(c *C) { "state": s.cr.State, "command": s.cr.Command, "output_path": s.cr.OutputPath, + "output_glob": s.cr.OutputGlob, "container_image": s.cr.ContainerImage, "mounts": s.cr.Mounts, "runtime_constraints": s.cr.RuntimeConstraints, @@ -158,11 +163,13 @@ func (s *integrationSuite) setup(c *C) { func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) { s.engine = "docker" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: docker Engine \d+\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) { s.engine = "singularity" s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: singularity.* version 3\.\d+.*`) } func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) { @@ -187,6 +194,16 @@ func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) { volume.AccessViaHosts = nil volume.Replication = 2 cluster.Volumes[uuid] = volume + + var v struct { + Root string + } + err = json.Unmarshal(volume.DriverParameters, &v) + c.Assert(err, IsNil) + err = os.Mkdir(v.Root, 0777) + if !os.IsExist(err) { + c.Assert(err, IsNil) + } } cluster.Containers.LocalKeepLogsToContainerLog = trial.logConfig @@ -201,37 +218,92 @@ func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) { s.engine = "docker" s.testRunTrivialContainer(c) - fs, err := s.logCollection.FileSystem(s.client, s.kc) - c.Assert(err, IsNil) - f, err := fs.Open("keepstore.txt") + log, logExists := s.logFiles["keepstore.txt"] if trial.logConfig == "none" { - c.Check(err, NotNil) - c.Check(os.IsNotExist(err), Equals, true) + c.Check(logExists, Equals, false) } else { - c.Assert(err, IsNil) - buf, err := ioutil.ReadAll(f) - c.Assert(err, IsNil) - c.Check(string(buf), trial.matchGetReq, `(?ms).*"reqMethod":"GET".*`) - c.Check(string(buf), trial.matchPutReq, `(?ms).*"reqMethod":"PUT".*,"reqPath":"0e3bcff26d51c895a60ea0d4585e134d".*`) + c.Check(log, Matches, `(?ms).*not running trash worker.*`) + c.Check(log, Matches, `(?ms).*not running trash emptier.*`) + c.Check(log, trial.matchGetReq, `(?ms).*"reqMethod":"GET".*`) + c.Check(log, trial.matchPutReq, `(?ms).*"reqMethod":"PUT".*,"reqPath":"0e3bcff26d51c895a60ea0d4585e134d".*`) } + + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*using local keepstore process .* at http://[\d\.]{7,}:\d+.*`) + c.Check(s.logFiles["crunch-run.txt"], Not(Matches), `(?ms).* at http://127\..*`) + c.Check(s.logFiles["crunch-run.txt"], Not(Matches), `(?ms).* at http://169\.254\..*`) + c.Check(s.logFiles["stderr.txt"], Matches, `(?ms).*ARVADOS_KEEP_SERVICES=http://[\d\.]{7,}:\d+\n.*`) } } +func (s *integrationSuite) TestRunTrivialContainerWithNoLocalKeepstore(c *C) { + // Check that (1) config is loaded from $ARVADOS_CONFIG when + // not provided on stdin and (2) if a local keepstore is not + // started, crunch-run.txt explains why not. + s.SetUpTest(c) + s.stdin.Reset() + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because KeepBuffers=0 in config\n.*`) + + s.SetUpTest(c) + s.args = []string{"-config", c.MkDir() + "/config.yaml"} + s.stdin.Reset() + buf, err := ioutil.ReadFile(os.Getenv("ARVADOS_CONFIG")) + c.Assert(err, IsNil) + err = ioutil.WriteFile(s.args[1], bytes.Replace(buf, []byte("LocalKeepBlobBuffersPerVCPU: 0"), []byte("LocalKeepBlobBuffersPerVCPU: 1"), -1), 0666) + c.Assert(err, IsNil) + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because a volume \(zzzzz-nyw5e-00000000000000\d\) uses AccessViaHosts\n.*`) + + // Check that config read errors are logged + s.SetUpTest(c) + s.args = []string{"-config", c.MkDir() + "/config-error.yaml"} + s.stdin.Reset() + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* no such file or directory\n.*`) + + s.SetUpTest(c) + s.args = []string{"-config", c.MkDir() + "/config-unreadable.yaml"} + s.stdin.Reset() + err = ioutil.WriteFile(s.args[1], []byte{}, 0) + c.Check(err, IsNil) + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* permission denied\n.*`) + + s.SetUpTest(c) + s.stdin.Reset() + s.testRunTrivialContainer(c) + c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*loaded config file \Q`+os.Getenv("ARVADOS_CONFIG")+`\E\n.*`) +} + +func (s *integrationSuite) TestRunTrivialContainerWithOutputGlob(c *C) { + s.cr.OutputGlob = []string{"js?n"} + s.testRunTrivialContainer(c) + fs, err := s.outputCollection.FileSystem(s.client, s.kc) + c.Assert(err, IsNil) + _, err = fs.Stat("json") + c.Check(err, IsNil) + _, err = fs.Stat("inputfile") + c.Check(err, Equals, os.ErrNotExist) + _, err = fs.Stat("emptydir") + c.Check(err, Equals, os.ErrNotExist) +} + func (s *integrationSuite) testRunTrivialContainer(c *C) { if err := exec.Command("which", s.engine).Run(); err != nil { c.Skip(fmt.Sprintf("%s: %s", s.engine, err)) } - s.cr.Command = []string{"sh", "-c", "cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"} + s.cr.Command = []string{"sh", "-c", "env >&2 && cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"} s.setup(c) args := []string{ "-runtime-engine=" + s.engine, "-enable-memory-limit=false", - s.cr.ContainerUUID, } if s.stdin.Len() > 0 { - args = append([]string{"-stdin-config=true"}, args...) + args = append(args, "-stdin-config=true") } + args = append(args, s.args...) + args = append(args, s.cr.ContainerUUID) code := command{}.RunCommand("crunch-run", args, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr)) c.Logf("\n===== stdout =====\n%s", s.stdout.String()) c.Logf("\n===== stderr =====\n%s", s.stderr.String()) @@ -257,6 +329,7 @@ func (s *integrationSuite) testRunTrivialContainer(c *C) { buf, err := ioutil.ReadAll(f) c.Assert(err, IsNil) c.Logf("\n===== %s =====\n%s", fi.Name(), buf) + s.logFiles[fi.Name()] = string(buf) } } s.logCollection = log @@ -264,34 +337,37 @@ func (s *integrationSuite) testRunTrivialContainer(c *C) { var output arvados.Collection err = s.client.RequestAndDecode(&output, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil) c.Assert(err, IsNil) - fs, err = output.FileSystem(s.client, s.kc) - c.Assert(err, IsNil) - if f, err := fs.Open("inputfile"); c.Check(err, IsNil) { - defer f.Close() - buf, err := ioutil.ReadAll(f) - c.Check(err, IsNil) - c.Check(string(buf), Equals, "inputdata") - } - if f, err := fs.Open("json"); c.Check(err, IsNil) { - defer f.Close() - buf, err := ioutil.ReadAll(f) - c.Check(err, IsNil) - c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`) - } - if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) { - c.Check(fi.IsDir(), Equals, true) - } - if d, err := fs.Open("emptydir"); c.Check(err, IsNil) { - defer d.Close() - fis, err := d.Readdir(-1) + s.outputCollection = output + + if len(s.cr.OutputGlob) == 0 { + fs, err = output.FileSystem(s.client, s.kc) c.Assert(err, IsNil) - // crunch-run still saves a ".keep" file to preserve - // empty dirs even though that shouldn't be - // necessary. Ideally we would do: - // c.Check(fis, HasLen, 0) - for _, fi := range fis { - c.Check(fi.Name(), Equals, ".keep") + if f, err := fs.Open("inputfile"); c.Check(err, IsNil) { + defer f.Close() + buf, err := ioutil.ReadAll(f) + c.Check(err, IsNil) + c.Check(string(buf), Equals, "inputdata") + } + if f, err := fs.Open("json"); c.Check(err, IsNil) { + defer f.Close() + buf, err := ioutil.ReadAll(f) + c.Check(err, IsNil) + c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`) + } + if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) { + c.Check(fi.IsDir(), Equals, true) + } + if d, err := fs.Open("emptydir"); c.Check(err, IsNil) { + defer d.Close() + fis, err := d.Readdir(-1) + c.Assert(err, IsNil) + // crunch-run still saves a ".keep" file to preserve + // empty dirs even though that shouldn't be + // necessary. Ideally we would do: + // c.Check(fis, HasLen, 0) + for _, fi := range fis { + c.Check(fi.Name(), Equals, ".keep") + } } } - s.outputCollection = output }