1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
17 "git.arvados.org/arvados.git/lib/config"
18 "git.arvados.org/arvados.git/sdk/go/arvados"
19 "git.arvados.org/arvados.git/sdk/go/arvadosclient"
20 "git.arvados.org/arvados.git/sdk/go/arvadostest"
21 "git.arvados.org/arvados.git/sdk/go/ctxlog"
22 "git.arvados.org/arvados.git/sdk/go/keepclient"
26 var _ = Suite(&integrationSuite{})
28 type integrationSuite struct {
30 image arvados.Collection
31 input arvados.Collection
36 cr arvados.ContainerRequest
37 client *arvados.Client
38 ac *arvadosclient.ArvadosClient
39 kc *keepclient.KeepClient
41 logCollection arvados.Collection
42 outputCollection arvados.Collection
43 logFiles map[string]string // filename => contents
46 func (s *integrationSuite) SetUpSuite(c *C) {
47 _, err := exec.LookPath("docker")
49 c.Skip("looks like docker is not installed")
52 arvadostest.StartKeep(2, true)
54 out, err := exec.Command("docker", "load", "--input", arvadostest.BusyboxDockerImage(c)).CombinedOutput()
57 out, err = exec.Command("arv-keepdocker", "--no-resume", "busybox:uclibc").Output()
58 imageUUID := strings.TrimSpace(string(out))
59 c.Logf("image uuid %s", imageUUID)
60 if !c.Check(err, IsNil) {
61 if err, ok := err.(*exec.ExitError); ok {
62 c.Logf("%s", err.Stderr)
66 err = arvados.NewClientFromEnv().RequestAndDecode(&s.image, "GET", "arvados/v1/collections/"+imageUUID, nil, nil)
68 c.Logf("image pdh %s", s.image.PortableDataHash)
70 s.client = arvados.NewClientFromEnv()
71 s.ac, err = arvadosclient.New(s.client)
73 s.kc = keepclient.New(s.ac)
74 fs, err := s.input.FileSystem(s.client, s.kc)
76 f, err := fs.OpenFile("inputfile", os.O_CREATE|os.O_WRONLY, 0755)
78 _, err = f.Write([]byte("inputdata"))
82 s.input.ManifestText, err = fs.MarshalManifest(".")
84 err = s.client.RequestAndDecode(&s.input, "POST", "arvados/v1/collections", nil, map[string]interface{}{
85 "ensure_unique_name": true,
86 "collection": map[string]interface{}{
87 "manifest_text": s.input.ManifestText,
91 c.Logf("input pdh %s", s.input.PortableDataHash)
94 func (s *integrationSuite) TearDownSuite(c *C) {
95 os.Unsetenv("ARVADOS_KEEP_SERVICES")
100 err := s.client.RequestAndDecode(nil, "POST", "database/reset", nil, nil)
104 func (s *integrationSuite) SetUpTest(c *C) {
105 os.Unsetenv("ARVADOS_KEEP_SERVICES")
108 s.stdin = bytes.Buffer{}
109 s.stdout = bytes.Buffer{}
110 s.stderr = bytes.Buffer{}
111 s.logCollection = arvados.Collection{}
112 s.outputCollection = arvados.Collection{}
113 s.logFiles = map[string]string{}
114 s.cr = arvados.ContainerRequest{
117 OutputPath: "/mnt/out",
118 ContainerImage: s.image.PortableDataHash,
119 Mounts: map[string]arvados.Mount{
122 Content: []interface{}{
124 map[string]string{"foo": "bar"},
130 PortableDataHash: s.input.PortableDataHash,
137 RuntimeConstraints: arvados.RuntimeConstraints{
145 func (s *integrationSuite) setup(c *C) {
146 err := s.client.RequestAndDecode(&s.cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
147 "priority": s.cr.Priority,
149 "command": s.cr.Command,
150 "output_path": s.cr.OutputPath,
151 "container_image": s.cr.ContainerImage,
152 "mounts": s.cr.Mounts,
153 "runtime_constraints": s.cr.RuntimeConstraints,
154 "use_existing": false,
157 c.Assert(s.cr.ContainerUUID, Not(Equals), "")
158 err = s.client.RequestAndDecode(nil, "POST", "arvados/v1/containers/"+s.cr.ContainerUUID+"/lock", nil, nil)
162 func (s *integrationSuite) TestRunTrivialContainerWithDocker(c *C) {
164 s.testRunTrivialContainer(c)
165 c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: docker Engine \d+\.\d+.*`)
168 func (s *integrationSuite) TestRunTrivialContainerWithSingularity(c *C) {
169 s.engine = "singularity"
170 s.testRunTrivialContainer(c)
171 c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*Using container runtime: singularity.* version 3\.\d+.*`)
174 func (s *integrationSuite) TestRunTrivialContainerWithLocalKeepstore(c *C) {
175 for _, trial := range []struct {
179 matchStartupMessage Checker
181 {"none", Not(Matches), Not(Matches), Not(Matches)},
182 {"all", Matches, Matches, Matches},
183 {"errors", Not(Matches), Not(Matches), Matches},
185 c.Logf("=== testing with Containers.LocalKeepLogsToContainerLog: %q", trial.logConfig)
188 cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
190 cluster, err := cfg.GetCluster("")
192 for uuid, volume := range cluster.Volumes {
193 volume.AccessViaHosts = nil
194 volume.Replication = 2
195 cluster.Volumes[uuid] = volume
200 err = json.Unmarshal(volume.DriverParameters, &v)
202 err = os.Mkdir(v.Root, 0777)
203 if !os.IsExist(err) {
207 cluster.Containers.LocalKeepLogsToContainerLog = trial.logConfig
210 err = json.NewEncoder(&s.stdin).Encode(ConfigData{
218 s.testRunTrivialContainer(c)
220 log, logExists := s.logFiles["keepstore.txt"]
221 if trial.logConfig == "none" {
222 c.Check(logExists, Equals, false)
224 c.Check(log, trial.matchGetReq, `(?ms).*"reqMethod":"GET".*`)
225 c.Check(log, trial.matchPutReq, `(?ms).*"reqMethod":"PUT".*,"reqPath":"0e3bcff26d51c895a60ea0d4585e134d".*`)
228 c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*using local keepstore process .* at http://[\d\.]{7,}:\d+.*`)
229 c.Check(s.logFiles["crunch-run.txt"], Not(Matches), `(?ms).* at http://127\..*`)
230 c.Check(s.logFiles["crunch-run.txt"], Not(Matches), `(?ms).* at http://169\.254\..*`)
231 c.Check(s.logFiles["stderr.txt"], Matches, `(?ms).*ARVADOS_KEEP_SERVICES=http://[\d\.]{7,}:\d+\n.*`)
235 func (s *integrationSuite) TestRunTrivialContainerWithNoLocalKeepstore(c *C) {
236 // Check that (1) config is loaded from $ARVADOS_CONFIG when
237 // not provided on stdin and (2) if a local keepstore is not
238 // started, crunch-run.txt explains why not.
241 s.testRunTrivialContainer(c)
242 c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because KeepBuffers=0 in config\n.*`)
245 s.args = []string{"-config", c.MkDir() + "/config.yaml"}
247 buf, err := ioutil.ReadFile(os.Getenv("ARVADOS_CONFIG"))
249 err = ioutil.WriteFile(s.args[1], bytes.Replace(buf, []byte("LocalKeepBlobBuffersPerVCPU: 0"), []byte("LocalKeepBlobBuffersPerVCPU: 1"), -1), 0666)
251 s.testRunTrivialContainer(c)
252 c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*not starting a local keepstore process because a volume \(zzzzz-nyw5e-00000000000000\d\) uses AccessViaHosts\n.*`)
254 // Check that config read errors are logged
256 s.args = []string{"-config", c.MkDir() + "/config-error.yaml"}
258 s.testRunTrivialContainer(c)
259 c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* no such file or directory\n.*`)
262 s.args = []string{"-config", c.MkDir() + "/config-unreadable.yaml"}
264 err = ioutil.WriteFile(s.args[1], []byte{}, 0)
266 s.testRunTrivialContainer(c)
267 c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*could not load config file \Q`+s.args[1]+`\E:.* permission denied\n.*`)
271 s.testRunTrivialContainer(c)
272 c.Check(s.logFiles["crunch-run.txt"], Matches, `(?ms).*loaded config file \Q`+os.Getenv("ARVADOS_CONFIG")+`\E\n.*`)
275 func (s *integrationSuite) testRunTrivialContainer(c *C) {
276 if err := exec.Command("which", s.engine).Run(); err != nil {
277 c.Skip(fmt.Sprintf("%s: %s", s.engine, err))
279 s.cr.Command = []string{"sh", "-c", "env >&2 && cat /mnt/in/inputfile >/mnt/out/inputfile && cat /mnt/json >/mnt/out/json && ! touch /mnt/in/shouldbereadonly && mkdir /mnt/out/emptydir"}
283 "-runtime-engine=" + s.engine,
284 "-enable-memory-limit=false",
286 if s.stdin.Len() > 0 {
287 args = append(args, "-stdin-config=true")
289 args = append(args, s.args...)
290 args = append(args, s.cr.ContainerUUID)
291 code := command{}.RunCommand("crunch-run", args, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr))
292 c.Logf("\n===== stdout =====\n%s", s.stdout.String())
293 c.Logf("\n===== stderr =====\n%s", s.stderr.String())
294 c.Check(code, Equals, 0)
295 err := s.client.RequestAndDecode(&s.cr, "GET", "arvados/v1/container_requests/"+s.cr.UUID, nil, nil)
297 c.Logf("Finished container request: %#v", s.cr)
299 var log arvados.Collection
300 err = s.client.RequestAndDecode(&log, "GET", "arvados/v1/collections/"+s.cr.LogUUID, nil, nil)
302 fs, err := log.FileSystem(s.client, s.kc)
304 if d, err := fs.Open("/"); c.Check(err, IsNil) {
305 fis, err := d.Readdir(-1)
307 for _, fi := range fis {
311 f, err := fs.Open(fi.Name())
313 buf, err := ioutil.ReadAll(f)
315 c.Logf("\n===== %s =====\n%s", fi.Name(), buf)
316 s.logFiles[fi.Name()] = string(buf)
319 s.logCollection = log
321 var output arvados.Collection
322 err = s.client.RequestAndDecode(&output, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil)
324 fs, err = output.FileSystem(s.client, s.kc)
326 if f, err := fs.Open("inputfile"); c.Check(err, IsNil) {
328 buf, err := ioutil.ReadAll(f)
330 c.Check(string(buf), Equals, "inputdata")
332 if f, err := fs.Open("json"); c.Check(err, IsNil) {
334 buf, err := ioutil.ReadAll(f)
336 c.Check(string(buf), Equals, `["foo",{"foo":"bar"},null]`)
338 if fi, err := fs.Stat("emptydir"); c.Check(err, IsNil) {
339 c.Check(fi.IsDir(), Equals, true)
341 if d, err := fs.Open("emptydir"); c.Check(err, IsNil) {
343 fis, err := d.Readdir(-1)
345 // crunch-run still saves a ".keep" file to preserve
346 // empty dirs even though that shouldn't be
347 // necessary. Ideally we would do:
348 // c.Check(fis, HasLen, 0)
349 for _, fi := range fis {
350 c.Check(fi.Name(), Equals, ".keep")
353 s.outputCollection = output