arvadostest.StartKeep(2, true)
- out, err := exec.Command("docker", "load", "--input", busyboxDockerImage(c)).CombinedOutput()
+ out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput()
c.Log(string(out))
c.Assert(err, IsNil)
out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output()
"state": s.cr.State,
"command": s.cr.Command,
"output_path": s.cr.OutputPath,
+ "output_glob": s.cr.OutputGlob,
"container_image": s.cr.ContainerImage,
"mounts": s.cr.Mounts,
"runtime_constraints": s.cr.RuntimeConstraints,
volume.AccessViaHosts = nil
volume.Replication = 2
cluster.Volumes[uuid] = volume
+
+ var v struct {
+ Root string
+ }
+ err = json.Unmarshal(volume.DriverParameters, &v)
+ c.Assert(err, IsNil)
+ err = os.Mkdir(v.Root, 0777)
+ if !os.IsExist(err) {
+ c.Assert(err, IsNil)
+ }
}
cluster.Containers.LocalKeepLogsToContainerLog = trial.logConfig
if trial.logConfig == "none" {
c.Check(logExists, Equals, false)
} else {
+ c.Check(log, Matches, `(?ms).*not running trash worker.*`)
+ c.Check(log, Matches, `(?ms).*not running trash emptier.*`)
c.Check(log, trial.matchGetReq, `(?ms).*"reqMethod":"GET".*`)
c.Check(log, trial.matchPutReq, `(?ms).*"reqMethod":"PUT".*,"reqPath":"0e3bcff26d51c895a60ea0d4585e134d".*`)
}
+
+ c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*using local keepstore process .* at http://[\d\.]{7,}:\d+.*`)
+ c.Check(s.logFiles["crunch-run.txt"], Not(Matches), `(?ms).* at http://127\..*`)
+ c.Check(s.logFiles["crunch-run.txt"], Not(Matches), `(?ms).* at http://169\.254\..*`)
+ c.Check(s.logFiles["stderr.txt"], Matches, `(?ms).*ARVADOS_KEEP_SERVICES=http://[\d\.]{7,}:\d+\n.*`)
}
+}
+func (s *integrationSuite) TestRunTrivialContainerWithNoLocalKeepstore(c *C) {
// Check that (1) config is loaded from $ARVADOS_CONFIG when
// not provided on stdin and (2) if a local keepstore is not
// started, crunch-run.txt explains why not.
s.SetUpTest(c)
s.stdin.Reset()
s.testRunTrivialContainer(c)
- c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because a volume \(zzzzz-nyw5e-000000000000000\) uses AccessViaHosts\n.*`)
+ c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because KeepBuffers=0 in config\n.*`)
+
+ s.SetUpTest(c)
+ s.args = []string{"-config", c.MkDir() + "/config.yaml"}
+ s.stdin.Reset()
+ buf, err := ioutil.ReadFile(os.Getenv("ARVADOS_CONFIG"))
+ c.Assert(err, IsNil)
+ err = ioutil.WriteFile(s.args[1], bytes.Replace(buf, []byte("LocalKeepBlobBuffersPerVCPU: 0"), []byte("LocalKeepBlobBuffersPerVCPU: 1"), -1), 0666)
+ c.Assert(err, IsNil)
+ s.testRunTrivialContainer(c)
+ c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because a volume \(zzzzz-nyw5e-00000000000000\d\) uses AccessViaHosts\n.*`)
// Check that config read errors are logged
s.SetUpTest(c)
s.SetUpTest(c)
s.args = []string{"-config", c.MkDir() + "/config-unreadable.yaml"}
s.stdin.Reset()
- err := ioutil.WriteFile(s.args[1], []byte{}, 0)
+ err = ioutil.WriteFile(s.args[1], []byte{}, 0)
c.Check(err, IsNil)
s.testRunTrivialContainer(c)
c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* permission denied\n.*`)
c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*loaded config file \Q`+os.Getenv("ARVADOS_CONFIG")+`\E\n.*`)
}
+func (s *integrationSuite) TestRunTrivialContainerWithOutputGlob(c *C) {
+ s.cr.OutputGlob = []string{"js?n"}
+ s.testRunTrivialContainer(c)
+ fs, err := s.outputCollection.FileSystem(s.client, s.kc)
+ c.Assert(err, IsNil)
+ _, err = fs.Stat("json")
+ c.Check(err, IsNil)
+ _, err = fs.Stat("inputfile")
+ c.Check(err, Equals, os.ErrNotExist)
+ _, err = fs.Stat("emptydir")
+ c.Check(err, Equals, os.ErrNotExist)
+}
+
func (s *integrationSuite) testRunTrivialContainer(c *C) {
if err := exec.Command("which", s.engine).Run(); err != nil {
c.Skip(fmt.Sprintf("%s: %s", s.engine, err))
}
- s.cr.Command = []string{"sh", "-c", "cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"}
+ s.cr.Command = []string{"sh", "-c", "env >&2 && cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"}
s.setup(c)
args := []string{
var output arvados.Collection
err = s.client.RequestAndDecode(&output, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil)
c.Assert(err, IsNil)
- fs, err = output.FileSystem(s.client, s.kc)
- c.Assert(err, IsNil)
- if f, err := fs.Open("inputfile"); c.Check(err, IsNil) {
- defer f.Close()
- buf, err := ioutil.ReadAll(f)
- c.Check(err, IsNil)
- c.Check(string(buf), Equals, "inputdata")
- }
- if f, err := fs.Open("json"); c.Check(err, IsNil) {
- defer f.Close()
- buf, err := ioutil.ReadAll(f)
- c.Check(err, IsNil)
- c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`)
- }
- if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) {
- c.Check(fi.IsDir(), Equals, true)
- }
- if d, err := fs.Open("emptydir"); c.Check(err, IsNil) {
- defer d.Close()
- fis, err := d.Readdir(-1)
+ s.outputCollection = output
+
+ if len(s.cr.OutputGlob) == 0 {
+ fs, err = output.FileSystem(s.client, s.kc)
c.Assert(err, IsNil)
- // crunch-run still saves a ".keep" file to preserve
- // empty dirs even though that shouldn't be
- // necessary. Ideally we would do:
- // c.Check(fis, HasLen, 0)
- for _, fi := range fis {
- c.Check(fi.Name(), Equals, ".keep")
+ if f, err := fs.Open("inputfile"); c.Check(err, IsNil) {
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, IsNil)
+ c.Check(string(buf), Equals, "inputdata")
+ }
+ if f, err := fs.Open("json"); c.Check(err, IsNil) {
+ defer f.Close()
+ buf, err := ioutil.ReadAll(f)
+ c.Check(err, IsNil)
+ c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`)
+ }
+ if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) {
+ c.Check(fi.IsDir(), Equals, true)
+ }
+ if d, err := fs.Open("emptydir"); c.Check(err, IsNil) {
+ defer d.Close()
+ fis, err := d.Readdir(-1)
+ c.Assert(err, IsNil)
+ // crunch-run still saves a ".keep" file to preserve
+ // empty dirs even though that shouldn't be
+ // necessary. Ideally we would do:
+ // c.Check(fis, HasLen, 0)
+ for _, fi := range fis {
+ c.Check(fi.Name(), Equals, ".keep")
+ }
}
}
- s.outputCollection = output
}
projects / arvados.git / blobdiff