X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/bb1d0c7499ad7c8d48c28a611c2d6a99c170265b..5c8b4fc39287a0075fc71c65684610fb4d623218:/services/crunch-run/crunchrun_test.go diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go index d3c9990770..4702838362 100644 --- a/services/crunch-run/crunchrun_test.go +++ b/services/crunch-run/crunchrun_test.go @@ -131,7 +131,7 @@ func (t *TestDockerClient) ContainerCreate(ctx context.Context, config *dockerco func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error { if container == "abcde" { - go t.fn(t) + // t.fn gets executed in ContainerWait return nil } else { return errors.New("Invalid container id") @@ -147,6 +147,7 @@ func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, body := make(chan dockercontainer.ContainerWaitOKBody) err := make(chan error) go func() { + t.fn(t) body <- dockercontainer.ContainerWaitOKBody{StatusCode: int64(t.finish)} close(body) close(err) @@ -155,6 +156,10 @@ func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, } func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) { + if t.finish == 2 { + return dockertypes.ImageInspect{}, nil, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?") + } + if t.imageLoaded == image { return dockertypes.ImageInspect{}, nil, nil } else { @@ -163,6 +168,9 @@ func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string } func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) { + if t.finish == 2 { + return dockertypes.ImageLoadResponse{}, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?") + } _, err := io.Copy(ioutil.Discard, input) if err != nil { return dockertypes.ImageLoadResponse{}, err @@ -667,9 +675,10 @@ func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn f if api.CalledWith("container.state", "Complete") != nil { c.Check(err, IsNil) } - c.Check(api.WasSetRunning, Equals, true) - - c.Check(api.Content[api.Calls-1]["container"].(arvadosclient.Dict)["log"], NotNil) + if exitCode != 2 { + c.Check(api.WasSetRunning, Equals, true) + c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil) + } if err != nil { for k, v := range api.Logs { @@ -1002,11 +1011,14 @@ func (s *TestSuite) TestSetupMounts(c *C) { cr.Container.Mounts = make(map[string]arvados.Mount) cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"} cr.OutputPath = "/tmp" - + cr.statInterval = 5 * time.Second err := cr.SetupMounts() c.Check(err, IsNil) - c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) + c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", + "--read-write", "--crunchstat-interval=5", + "--mount-by-pdh", "by_id", realTemp + "/keep1"}) c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp"}) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1021,8 +1033,11 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, IsNil) - c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) + c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", + "--read-write", "--crunchstat-interval=5", + "--mount-by-pdh", "by_id", realTemp + "/keep1"}) c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/out", realTemp + "/3:/tmp"}) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1039,8 +1054,11 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, IsNil) - c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) + c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", + "--read-write", "--crunchstat-interval=5", + "--mount-by-pdh", "by_id", realTemp + "/keep1"}) c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"}) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() @@ -1059,8 +1077,11 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, IsNil) - c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) + c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", + "--read-write", "--crunchstat-interval=5", + "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/tmp0:/keeptmp"}) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1079,10 +1100,13 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, IsNil) - c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) + c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", + "--read-write", "--crunchstat-interval=5", + "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) sort.StringSlice(cr.Binds).Sort() c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro", realTemp + "/keep1/tmp0:/keepout"}) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1102,10 +1126,13 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, IsNil) - c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) + c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", + "--read-write", "--crunchstat-interval=5", + "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) sort.StringSlice(cr.Binds).Sort() c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro", realTemp + "/keep1/tmp0:/keepout"}) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1130,6 +1157,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { content, err := ioutil.ReadFile(realTemp + "/2/mountdata.json") c.Check(err, IsNil) c.Check(content, DeepEquals, []byte(test.out)) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1149,8 +1177,11 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, IsNil) - c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) + c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", + "--read-write", "--crunchstat-interval=5", + "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"}) c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"}) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1169,6 +1200,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, NotNil) c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1187,6 +1219,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, NotNil) c.Check(err, ErrorMatches, `Only mount points of kind 'collection' are supported underneath the output_path.*`) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1203,6 +1236,7 @@ func (s *TestSuite) TestSetupMounts(c *C) { err := cr.SetupMounts() c.Check(err, NotNil) c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`) + os.RemoveAll(cr.ArvMountPoint) cr.CleanupDirs() checkEmpty() } @@ -1733,3 +1767,61 @@ func (s *TestSuite) TestEvalSymlinkDir(c *C) { _, err = cr.UploadOutputFile(realTemp+"/"+v, info, err, []string{}, nil, "", "", 0) c.Assert(err, NotNil) } + +func (s *TestSuite) TestFullBrokenDocker1(c *C) { + tf, err := ioutil.TempFile("", "brokenNodeHook-") + c.Assert(err, IsNil) + defer os.Remove(tf.Name()) + + tf.Write([]byte(`#!/bin/sh +exec echo killme +`)) + tf.Close() + os.Chmod(tf.Name(), 0700) + + ech := tf.Name() + brokenNodeHook = &ech + + api, _, _ := FullRunHelper(c, `{ + "command": ["echo", "hello world"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "environment": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {} +}`, nil, 2, func(t *TestDockerClient) { + t.logWriter.Write(dockerLog(1, "hello world\n")) + t.logWriter.Close() + }) + + c.Check(api.CalledWith("container.state", "Queued"), NotNil) + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*") + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*") + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*") + +} + +func (s *TestSuite) TestFullBrokenDocker2(c *C) { + ech := "" + brokenNodeHook = &ech + + api, _, _ := FullRunHelper(c, `{ + "command": ["echo", "hello world"], + "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122", + "cwd": ".", + "environment": {}, + "mounts": {"/tmp": {"kind": "tmp"} }, + "output_path": "/tmp", + "priority": 1, + "runtime_constraints": {} +}`, nil, 2, func(t *TestDockerClient) { + t.logWriter.Write(dockerLog(1, "hello world\n")) + t.logWriter.Close() + }) + + c.Check(api.CalledWith("container.state", "Queued"), NotNil) + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*") + c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*No broken node hook.*") +}