X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b7b9ea44ada30b1251fb10c872cb1da1d7c29bd0..36352ce56096e4406344c531147236c355c73235:/services/crunchstat/crunchstat_test.go

diff --git a/services/crunchstat/crunchstat_test.go b/services/crunchstat/crunchstat_test.go
index fe3b56d258..759b3aa073 100644
--- a/services/crunchstat/crunchstat_test.go
+++ b/services/crunchstat/crunchstat_test.go
@@ -3,9 +3,15 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"math/rand"
+	"os"
+	"os/exec"
+	"sync"
+	"syscall"
 	"testing"
 	"time"
 )
@@ -82,3 +88,147 @@ func bufLogger() (*log.Logger, *bufio.Reader) {
 	logger := log.New(w, "", 0)
 	return logger, bufio.NewReader(r)
 }
+
+func TestSignalOnDeadPPID(t *testing.T) {
+	if !testDeadParent(t, 0) {
+		t.Fatal("child should still be alive after parent dies")
+	}
+	if testDeadParent(t, 15) {
+		t.Fatal("child should have been killed when parent died")
+	}
+}
+
+// testDeadParent returns true if crunchstat's child proc is still
+// alive after its parent dies.
+func testDeadParent(t *testing.T, signum int) bool {
+	var err error
+	var bin, childlockfile, parentlockfile *os.File
+	for _, f := range []**os.File{&bin, &childlockfile, &parentlockfile} {
+		*f, err = ioutil.TempFile("", "crunchstat_")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer (*f).Close()
+		defer os.Remove((*f).Name())
+	}
+
+	bin.Close()
+	err = exec.Command("go", "build", "-o", bin.Name()).Run()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = syscall.Flock(int(parentlockfile.Fd()), syscall.LOCK_EX)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cmd := exec.Command("bash", "-c", `
+set -e
+"$BINFILE" -cgroup-root=/none -ppid-check-interval=10ms -signal-on-dead-ppid="$SIGNUM" bash -c '
+    set -e
+    unlock() {
+        flock --unlock "$CHILDLOCKFD"
+        kill %1
+    }
+    trap unlock TERM
+    flock --exclusive "$CHILDLOCKFD"
+    echo -n "$$" > "$CHILDLOCKFILE"
+    flock --unlock "$PARENTLOCKFD"
+    sleep 20 </dev/null >/dev/null 2>/dev/null &
+    wait %1
+    unlock
+' &
+
+# wait for inner bash to start, to ensure $BINFILE has seen this bash proc as its initial PPID
+flock --exclusive "$PARENTLOCKFILE" true
+`)
+	cmd.Env = append(os.Environ(),
+		"SIGNUM="+fmt.Sprintf("%d", signum),
+		"PARENTLOCKFD=3",
+		"PARENTLOCKFILE="+parentlockfile.Name(),
+		"CHILDLOCKFD=4",
+		"CHILDLOCKFILE="+childlockfile.Name(),
+		"BINFILE="+bin.Name())
+	cmd.ExtraFiles = []*os.File{parentlockfile, childlockfile}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cmd.Start()
+	defer cmd.Wait()
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	defer wg.Wait()
+	for _, rdr := range []io.ReadCloser{stderr, stdout} {
+		go func(rdr io.ReadCloser) {
+			defer wg.Done()
+			buf := make([]byte, 1024)
+			for {
+				n, err := rdr.Read(buf)
+				if n > 0 {
+					t.Logf("%s", buf[:n])
+				}
+				if err != nil {
+					return
+				}
+			}
+		}(rdr)
+	}
+
+	// Wait until inner bash process releases parentlockfile
+	// (which means it has locked childlockfile and written its
+	// PID)
+	err = exec.Command("flock", "--exclusive", parentlockfile.Name(), "true").Run()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	childDone := make(chan bool)
+	go func() {
+		// Notify the main thread when the inner bash process
+		// releases its lock on childlockfile (which means
+		// either its sleep process ended or it received a
+		// TERM signal).
+		t0 := time.Now()
+		err = exec.Command("flock", "--exclusive", childlockfile.Name(), "true").Run()
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Logf("child done after %s", time.Since(t0))
+		close(childDone)
+	}()
+
+	select {
+	case <-time.After(500 * time.Millisecond):
+		// Inner bash process is still alive after the timeout
+		// period. Kill it now, so our stdout and stderr pipes
+		// can finish and we don't leave a mess of child procs
+		// behind.
+		buf, err := ioutil.ReadFile(childlockfile.Name())
+		if err != nil {
+			t.Fatal(err)
+		}
+		var childPID int
+		_, err = fmt.Sscanf(string(buf), "%d", &childPID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		child, err := os.FindProcess(childPID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		child.Signal(syscall.Signal(15))
+		return true
+
+	case <-childDone:
+		// Inner bash process ended soon after its grandparent
+		// ended.
+		return false
+	}
+}