X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d3cef2f89fbcbf9bac380a7c834f13117d187058..8fc29fafb91cf64ce4ededbdd85ef9507c51f216:/services/crunch-run/background.go

diff --git a/services/crunch-run/background.go b/services/crunch-run/background.go
index 334b3ab674..852ccb6ece 100644
--- a/services/crunch-run/background.go
+++ b/services/crunch-run/background.go
@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -21,20 +20,22 @@ var (
 	lockdir    = "/var/lock"
 	lockprefix = "crunch-run-"
 	locksuffix = ".lock"
+	brokenfile = "crunch-run-broken"
 )
 
 // procinfo is saved in each process's lockfile.
 type procinfo struct {
-	UUID   string
-	PID    int
-	Stdout string
-	Stderr string
+	UUID string
+	PID  int
 }
 
 // Detach acquires a lock for the given uuid, and starts the current
 // program as a child process (with -no-detach prepended to the given
 // arguments so the child knows not to detach again). The lock is
 // passed along to the child process.
+//
+// Stdout and stderr in the child process are sent to the systemd
+// journal using the systemd-cat program.
 func Detach(uuid string, args []string, stdout, stderr io.Writer) int {
 	return exitcode(stderr, detach(uuid, args, stdout, stderr))
 }
@@ -67,21 +68,7 @@ func detach(uuid string, args []string, stdout, stderr io.Writer) error {
 	defer lockfile.Close()
 	lockfile.Truncate(0)
 
-	outfile, err := ioutil.TempFile("", "crunch-run-"+uuid+"-stdout-")
-	if err != nil {
-		return err
-	}
-	defer outfile.Close()
-	errfile, err := ioutil.TempFile("", "crunch-run-"+uuid+"-stderr-")
-	if err != nil {
-		os.Remove(outfile.Name())
-		return err
-	}
-	defer errfile.Close()
-
-	cmd := exec.Command(args[0], append([]string{"-no-detach"}, args[1:]...)...)
-	cmd.Stdout = outfile
-	cmd.Stderr = errfile
+	cmd := exec.Command("systemd-cat", append([]string{"--identifier=crunch-run", args[0], "-no-detach"}, args[1:]...)...)
 	// Child inherits lockfile.
 	cmd.ExtraFiles = []*os.File{lockfile}
 	// Ensure child isn't interrupted even if we receive signals
@@ -90,24 +77,14 @@ func detach(uuid string, args []string, stdout, stderr io.Writer) error {
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	err = cmd.Start()
 	if err != nil {
-		os.Remove(outfile.Name())
-		os.Remove(errfile.Name())
 		return fmt.Errorf("exec %s: %s", cmd.Path, err)
 	}
 
 	w := io.MultiWriter(stdout, lockfile)
-	err = json.NewEncoder(w).Encode(procinfo{
-		UUID:   uuid,
-		PID:    cmd.Process.Pid,
-		Stdout: outfile.Name(),
-		Stderr: errfile.Name(),
+	return json.NewEncoder(w).Encode(procinfo{
+		UUID: uuid,
+		PID:  cmd.Process.Pid,
 	})
-	if err != nil {
-		os.Remove(outfile.Name())
-		os.Remove(errfile.Name())
-		return err
-	}
-	return nil
 }
 
 // KillProcess finds the crunch-run process corresponding to the given
@@ -140,14 +117,21 @@ func kill(uuid string, signal syscall.Signal, stdout, stderr io.Writer) error {
 
 	proc, err := os.FindProcess(pi.PID)
 	if err != nil {
+		// FindProcess should have succeeded, even if the
+		// process does not exist.
 		return fmt.Errorf("%s: find process %d: %s", uuid, pi.PID, err)
 	}
 
+	// Send the requested signal once, then send signal 0 a few
+	// times.  When proc.Signal() returns an error (process no
+	// longer exists), return success.  If that doesn't happen
+	// within 1 second, return an error.
 	err = proc.Signal(signal)
 	for deadline := time.Now().Add(time.Second); err == nil && time.Now().Before(deadline); time.Sleep(time.Second / 100) {
 		err = proc.Signal(syscall.Signal(0))
 	}
 	if err == nil {
+		// Reached deadline without a proc.Signal() error.
 		return fmt.Errorf("%s: pid %d: sent signal %d (%s) but process is still alive", uuid, pi.PID, signal, signal)
 	}
 	fmt.Fprintf(stderr, "%s: pid %d: %s\n", uuid, pi.PID, err)
@@ -163,7 +147,10 @@ func ListProcesses(stdout, stderr io.Writer) int {
 		if info.IsDir() && path != walkdir {
 			return filepath.SkipDir
 		}
-		if name := info.Name(); !strings.HasPrefix(name, lockprefix) || !strings.HasSuffix(name, locksuffix) {
+		if name := info.Name(); name == brokenfile {
+			fmt.Fprintln(stdout, "broken")
+			return nil
+		} else if !strings.HasPrefix(name, lockprefix) || !strings.HasSuffix(name, locksuffix) {
 			return nil
 		}
 		if info.Size() == 0 {