X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/bb91f86ea02e3535e3953ee0916dd6877bf585f7..cbeb2a6dc91804addea68705579db7ed1d53458a:/lib/crunchstat/crunchstat.go

diff --git a/lib/crunchstat/crunchstat.go b/lib/crunchstat/crunchstat.go
index 056ef0d185..10cd7cfce4 100644
--- a/lib/crunchstat/crunchstat.go
+++ b/lib/crunchstat/crunchstat.go
@@ -17,19 +17,10 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 )
 
-// This magically allows us to look up userHz via _SC_CLK_TCK:
-
-/*
-#include <unistd.h>
-#include <sys/types.h>
-#include <pwd.h>
-#include <stdlib.h>
-*/
-import "C"
-
 // A Reporter gathers statistics for a cgroup and writes them to a
 // log.Logger.
 type Reporter struct {
@@ -52,13 +43,17 @@ type Reporter struct {
 	// Interval between samples. Must be positive.
 	PollPeriod time.Duration
 
+	// Temporary directory, will be monitored for available, used & total space.
+	TempDir string
+
 	// Where to write statistics. Must not be nil.
 	Logger *log.Logger
 
-	reportedStatFile map[string]string
-	lastNetSample    map[string]ioSample
-	lastDiskSample   map[string]ioSample
-	lastCPUSample    cpuSample
+	reportedStatFile    map[string]string
+	lastNetSample       map[string]ioSample
+	lastDiskIOSample    map[string]ioSample
+	lastCPUSample       cpuSample
+	lastDiskSpaceSample diskSpaceSample
 
 	done    chan struct{} // closed when we should stop reporting
 	flushed chan struct{} // closed when we have made our last report
@@ -216,14 +211,14 @@ func (r *Reporter) doBlkIOStats() {
 			continue
 		}
 		delta := ""
-		if prev, ok := r.lastDiskSample[dev]; ok {
+		if prev, ok := r.lastDiskIOSample[dev]; ok {
 			delta = fmt.Sprintf(" -- interval %.4f seconds %d write %d read",
 				sample.sampleTime.Sub(prev.sampleTime).Seconds(),
 				sample.txBytes-prev.txBytes,
 				sample.rxBytes-prev.rxBytes)
 		}
 		r.Logger.Printf("blkio:%s %d write %d read%s\n", dev, sample.txBytes, sample.rxBytes, delta)
-		r.lastDiskSample[dev] = sample
+		r.lastDiskIOSample[dev] = sample
 	}
 }
 
@@ -251,8 +246,13 @@ func (r *Reporter) doMemoryStats() {
 	}
 	var outstat bytes.Buffer
 	for _, key := range wantStats {
-		if val, ok := thisSample.memStat[key]; ok {
-			outstat.WriteString(fmt.Sprintf(" %d %s", val, key))
+		// Use "total_X" stats (entire hierarchy) if enabled,
+		// otherwise just the single cgroup -- see
+		// https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
+		if val, ok := thisSample.memStat["total_"+key]; ok {
+			fmt.Fprintf(&outstat, " %d %s", val, key)
+		} else if val, ok := thisSample.memStat[key]; ok {
+			fmt.Fprintf(&outstat, " %d %s", val, key)
 		}
 	}
 	r.Logger.Printf("mem%s\n", outstat.String())
@@ -302,6 +302,42 @@ func (r *Reporter) doNetworkStats() {
 	}
 }
 
+type diskSpaceSample struct {
+	hasData    bool
+	sampleTime time.Time
+	total      uint64
+	used       uint64
+	available  uint64
+}
+
+func (r *Reporter) doDiskSpaceStats() {
+	s := syscall.Statfs_t{}
+	err := syscall.Statfs(r.TempDir, &s)
+	if err != nil {
+		return
+	}
+	bs := uint64(s.Bsize)
+	nextSample := diskSpaceSample{
+		hasData:    true,
+		sampleTime: time.Now(),
+		total:      s.Blocks * bs,
+		used:       (s.Blocks - s.Bfree) * bs,
+		available:  s.Bavail * bs,
+	}
+
+	var delta string
+	if r.lastDiskSpaceSample.hasData {
+		prev := r.lastDiskSpaceSample
+		interval := nextSample.sampleTime.Sub(prev.sampleTime).Seconds()
+		delta = fmt.Sprintf(" -- interval %.4f seconds %d used",
+			interval,
+			int64(nextSample.used-prev.used))
+	}
+	r.Logger.Printf("statfs %d available %d used %d total%s\n",
+		nextSample.available, nextSample.used, nextSample.total, delta)
+	r.lastDiskSpaceSample = nextSample
+}
+
 type cpuSample struct {
 	hasData    bool // to distinguish the zero value from real data
 	sampleTime time.Time
@@ -349,7 +385,7 @@ func (r *Reporter) doCPUStats() {
 
 	var userTicks, sysTicks int64
 	fmt.Sscanf(string(b), "user %d\nsystem %d", &userTicks, &sysTicks)
-	userHz := float64(C.sysconf(C._SC_CLK_TCK))
+	userHz := float64(100)
 	nextSample := cpuSample{
 		hasData:    true,
 		sampleTime: time.Now(),
@@ -382,7 +418,15 @@ func (r *Reporter) run() {
 	}
 
 	r.lastNetSample = make(map[string]ioSample)
-	r.lastDiskSample = make(map[string]ioSample)
+	r.lastDiskIOSample = make(map[string]ioSample)
+
+	if len(r.TempDir) == 0 {
+		// Temporary dir not provided, try to get it from the environment.
+		r.TempDir = os.Getenv("TMPDIR")
+	}
+	if len(r.TempDir) > 0 {
+		r.Logger.Printf("notice: monitoring temp dir %s\n", r.TempDir)
+	}
 
 	ticker := time.NewTicker(r.PollPeriod)
 	for {
@@ -390,6 +434,7 @@ func (r *Reporter) run() {
 		r.doCPUStats()
 		r.doBlkIOStats()
 		r.doNetworkStats()
+		r.doDiskSpaceStats()
 		select {
 		case <-r.done:
 			return