12614: Add tests.

[arvados.git] / services / crunch-run / crunchrun_test.go

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index d3c9990770db7c6ac24f92b97fea3d82eb0e8814..6581c6bfb2b96245d83977a1473e7d81f9ec2b0d 100644 (file)
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -131,7 +131,7 @@ func (t *TestDockerClient) ContainerCreate(ctx context.Context, config *dockerco
 
 func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error {
        if container == "abcde" {
-               go t.fn(t)
+               // t.fn gets executed in ContainerWait
                return nil
        } else {
                return errors.New("Invalid container id")
@@ -147,6 +147,7 @@ func (t *TestDockerClient) ContainerWait(ctx context.Context, container string,
        body := make(chan dockercontainer.ContainerWaitOKBody)
        err := make(chan error)
        go func() {
+               t.fn(t)
                body <- dockercontainer.ContainerWaitOKBody{StatusCode: int64(t.finish)}
                close(body)
                close(err)
@@ -155,6 +156,10 @@ func (t *TestDockerClient) ContainerWait(ctx context.Context, container string,
 }
 
 func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
+       if t.finish == 2 {
+               return dockertypes.ImageInspect{}, nil, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
+       }
+
        if t.imageLoaded == image {
                return dockertypes.ImageInspect{}, nil, nil
        } else {
@@ -163,6 +168,9 @@ func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string
 }
 
 func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) {
+       if t.finish == 2 {
+               return dockertypes.ImageLoadResponse{}, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
+       }
        _, err := io.Copy(ioutil.Discard, input)
        if err != nil {
                return dockertypes.ImageLoadResponse{}, err
@@ -667,9 +675,10 @@ func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn f
        if api.CalledWith("container.state", "Complete") != nil {
                c.Check(err, IsNil)
        }
-       c.Check(api.WasSetRunning, Equals, true)
-
-       c.Check(api.Content[api.Calls-1]["container"].(arvadosclient.Dict)["log"], NotNil)
+       if exitCode != 2 {
+               c.Check(api.WasSetRunning, Equals, true)
+               c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil)
+       }
 
        if err != nil {
                for k, v := range api.Logs {
@@ -1002,11 +1011,14 @@ func (s *TestSuite) TestSetupMounts(c *C) {
                cr.Container.Mounts = make(map[string]arvados.Mount)
                cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
                cr.OutputPath = "/tmp"
-
+               cr.statInterval = 5 * time.Second
                err := cr.SetupMounts()
                c.Check(err, IsNil)
-               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
+                       "--read-write", "--crunchstat-interval=5",
+                       "--mount-by-pdh", "by_id", realTemp + "/keep1"})
                c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp"})
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1021,8 +1033,11 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 
                err := cr.SetupMounts()
                c.Check(err, IsNil)
-               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
+                       "--read-write", "--crunchstat-interval=5",
+                       "--mount-by-pdh", "by_id", realTemp + "/keep1"})
                c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/out", realTemp + "/3:/tmp"})
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1039,8 +1054,11 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 
                err := cr.SetupMounts()
                c.Check(err, IsNil)
-               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
+                       "--read-write", "--crunchstat-interval=5",
+                       "--mount-by-pdh", "by_id", realTemp + "/keep1"})
                c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"})
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
 
@@ -1059,8 +1077,11 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 
                err := cr.SetupMounts()
                c.Check(err, IsNil)
-               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
+                       "--read-write", "--crunchstat-interval=5",
+                       "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
                c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/tmp0:/keeptmp"})
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1079,10 +1100,13 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 
                err := cr.SetupMounts()
                c.Check(err, IsNil)
-               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
+                       "--read-write", "--crunchstat-interval=5",
+                       "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
                sort.StringSlice(cr.Binds).Sort()
                c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
                        realTemp + "/keep1/tmp0:/keepout"})
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1102,10 +1126,13 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 
                err := cr.SetupMounts()
                c.Check(err, IsNil)
-               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
+                       "--read-write", "--crunchstat-interval=5",
+                       "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
                sort.StringSlice(cr.Binds).Sort()
                c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
                        realTemp + "/keep1/tmp0:/keepout"})
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1130,6 +1157,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
                content, err := ioutil.ReadFile(realTemp + "/2/mountdata.json")
                c.Check(err, IsNil)
                c.Check(content, DeepEquals, []byte(test.out))
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1149,8 +1177,11 @@ func (s *TestSuite) TestSetupMounts(c *C) {
 
                err := cr.SetupMounts()
                c.Check(err, IsNil)
-               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
+               c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
+                       "--read-write", "--crunchstat-interval=5",
+                       "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
                c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"})
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1169,6 +1200,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
                err := cr.SetupMounts()
                c.Check(err, NotNil)
                c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`)
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1187,6 +1219,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
                err := cr.SetupMounts()
                c.Check(err, NotNil)
                c.Check(err, ErrorMatches, `Only mount points of kind 'collection' are supported underneath the output_path.*`)
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1203,6 +1236,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
                err := cr.SetupMounts()
                c.Check(err, NotNil)
                c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
+               os.RemoveAll(cr.ArvMountPoint)
                cr.CleanupDirs()
                checkEmpty()
        }
@@ -1733,3 +1767,50 @@ func (s *TestSuite) TestEvalSymlinkDir(c *C) {
        _, err = cr.UploadOutputFile(realTemp+"/"+v, info, err, []string{}, nil, "", "", 0)
        c.Assert(err, NotNil)
 }
+
+func (s *TestSuite) TestFullBrokenDocker1(c *C) {
+       api, _, _ := FullRunHelper(c, `{
+    "command": ["echo", "hello world"],
+    "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+    "cwd": ".",
+    "environment": {},
+    "mounts": {"/tmp": {"kind": "tmp"} },
+    "output_path": "/tmp",
+    "priority": 1,
+    "runtime_constraints": {}
+}`, nil, 2, func(t *TestDockerClient) {
+               t.logWriter.Write(dockerLog(1, "hello world\n"))
+               t.logWriter.Close()
+       })
+
+       ech := "/bin/echo"
+       brokenNodeHook = &ech
+
+       c.Check(api.CalledWith("container.state", "Queued"), NotNil)
+       c.Check(strings.Index(api.Logs["crunch-run"].String(), "unable to run containers"), Not(Equals), -1)
+       c.Check(strings.Index(api.Logs["crunch-run"].String(), "Running broken node hook '/bin/echo'"), Not(Equals), -1)
+
+}
+
+func (s *TestSuite) TestFullBrokenDocker2(c *C) {
+       api, _, _ := FullRunHelper(c, `{
+    "command": ["echo", "hello world"],
+    "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+    "cwd": ".",
+    "environment": {},
+    "mounts": {"/tmp": {"kind": "tmp"} },
+    "output_path": "/tmp",
+    "priority": 1,
+    "runtime_constraints": {}
+}`, nil, 2, func(t *TestDockerClient) {
+               t.logWriter.Write(dockerLog(1, "hello world\n"))
+               t.logWriter.Close()
+       })
+
+       ech := ""
+       brokenNodeHook = &ech
+
+       c.Check(api.CalledWith("container.state", "Queued"), NotNil)
+       c.Check(strings.Index(api.Logs["crunch-run"].String(), "unable to run containers"), Not(Equals), -1)
+       c.Check(strings.Index(api.Logs["crunch-run"].String(), "No broken node hook"), Not(Equals), -1)
+}