X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b435064a0085044993c5c0f5b8156a61b3e32c54..06d0338d638ce74b51ca3cfe10523e5515269fdf:/services/crunchstat/crunchstat.go diff --git a/services/crunchstat/crunchstat.go b/services/crunchstat/crunchstat.go index bd91aa0dec..91027d7677 100644 --- a/services/crunchstat/crunchstat.go +++ b/services/crunchstat/crunchstat.go @@ -3,8 +3,8 @@ package main import ( "bufio" "bytes" - "flag" "errors" + "flag" "fmt" "io" "io/ioutil" @@ -12,6 +12,7 @@ import ( "os" "os/exec" "os/signal" + "strconv" "strings" "syscall" "time" @@ -24,6 +25,7 @@ import ( #include */ import "C" + // The above block of magic allows us to look up user_hz via _SC_CLK_TCK. type Cgroup struct { @@ -46,70 +48,70 @@ func CopyChanToPipe(in <-chan string, out io.Writer) { } } -func OpenAndReadAll(filename string, log_chan chan<- string) ([]byte, error) { - in, err := os.Open(filename) - if err != nil { - if log_chan != nil { - log_chan <- fmt.Sprintf("crunchstat: open %s: %s", filename, err) - } - return nil, err +var logChan chan string +func LogPrintf(format string, args ...interface{}) { + if logChan == nil { + return } - defer in.Close() - { - content, err := ioutil.ReadAll(in) - if err != nil && log_chan != nil { - log_chan <- fmt.Sprintf("crunchstat: read %s: %s", filename, err) - } - return content, err + logChan <- fmt.Sprintf("crunchstat: " + format, args...) +} + +func ReadAllOrWarn(in *os.File) ([]byte, error) { + content, err := ioutil.ReadAll(in) + if err != nil { + LogPrintf("read %s: %s", in.Name(), err) } + return content, err } -var reportedStatFile map[string]bool -var reportedNoStatFile map[string]bool +var reportedStatFile = map[string]string{} -// Find the cgroup stats file in /sys/fs corresponding to the target -// cgroup. +// Open the cgroup stats file in /sys/fs corresponding to the target +// cgroup, and return an *os.File. If no stats file is available, +// return nil. // // TODO: Instead of trying all options, choose a process in the // container, and read /proc/PID/cgroup to determine the appropriate // cgroup root for the given statgroup. (This will avoid falling back // to host-level stats during container setup and teardown.) -func FindStat(stderr chan<- string, cgroup Cgroup, statgroup string, stat string) string { - if reportedStatFile == nil { - reportedStatFile = make(map[string]bool) - reportedNoStatFile = make(map[string]bool) +func OpenStatFile(cgroup Cgroup, statgroup string, stat string) (*os.File, error) { + var paths = []string{ + fmt.Sprintf("%s/%s/%s/%s/%s", cgroup.root, statgroup, cgroup.parent, cgroup.cid, stat), + fmt.Sprintf("%s/%s/%s/%s", cgroup.root, cgroup.parent, cgroup.cid, stat), + fmt.Sprintf("%s/%s/%s", cgroup.root, statgroup, stat), + fmt.Sprintf("%s/%s", cgroup.root, stat), } - var path string - path = fmt.Sprintf("%s/%s/%s/%s/%s", cgroup.root, statgroup, cgroup.parent, cgroup.cid, stat) - if _, err := os.Stat(path); err != nil { - path = fmt.Sprintf("%s/%s/%s/%s", cgroup.root, cgroup.parent, cgroup.cid, stat) - } - if _, err := os.Stat(path); err != nil { - path = fmt.Sprintf("%s/%s/%s", cgroup.root, statgroup, stat) - } - if _, err := os.Stat(path); err != nil { - path = fmt.Sprintf("%s/%s", cgroup.root, stat) - } - if _, err := os.Stat(path); err != nil { - if _, ok := reportedNoStatFile[stat]; !ok { - stderr <- fmt.Sprintf("crunchstat: did not find stats file (root %s, parent %s, cid %s, statgroup %s, stat %s)", cgroup.root, cgroup.parent, cgroup.cid, statgroup, stat) - reportedNoStatFile[stat] = true + var file *os.File + var err error + for _, path = range paths { + file, err = os.Open(path) + if err == nil { + break + } else { + path = "" } - return "" } - if _, ok := reportedStatFile[path]; !ok { - stderr <- fmt.Sprintf("crunchstat: reading stats from %s", path) - reportedStatFile[path] = true + if pathWas, ok := reportedStatFile[stat]; !ok || pathWas != path { + // Log whenever we start using a new/different cgroup + // stat file for a given statistic. This typically + // happens 1 to 3 times per statistic, depending on + // whether we happen to collect stats [a] before any + // processes have been created in the container and + // [b] after all contained processes have exited. + reportedStatFile[stat] = path + if path == "" { + LogPrintf("did not find stats file: stat %s, statgroup %s, cid %s, parent %s, root %s", stat, statgroup, cgroup.cid, cgroup.parent, cgroup.root) + } else { + LogPrintf("reading stats from %s", path) + } } - return path + return file, err } -func GetContainerNetStats(stderr chan<- string, cgroup Cgroup) (io.Reader, error) { - procsFilename := FindStat(stderr, cgroup, "cpuacct", "cgroup.procs") - procsFile, err := os.Open(procsFilename) +func GetContainerNetStats(cgroup Cgroup) (io.Reader, error) { + procsFile, err := OpenStatFile(cgroup, "cpuacct", "cgroup.procs") if err != nil { - stderr <- fmt.Sprintf("crunchstat: open %s: %s", procsFilename, err) return nil, err } defer procsFile.Close() @@ -117,8 +119,9 @@ func GetContainerNetStats(stderr chan<- string, cgroup Cgroup) (io.Reader, error for reader.Scan() { taskPid := reader.Text() statsFilename := fmt.Sprintf("/proc/%s/net/dev", taskPid) - stats, err := OpenAndReadAll(statsFilename, stderr) + stats, err := ioutil.ReadFile(statsFilename) if err != nil { + LogPrintf("read %s: %s", statsFilename, err) continue } return strings.NewReader(string(stats)), nil @@ -132,16 +135,10 @@ type IoSample struct { rxBytes int64 } -func DoBlkIoStats(stderr chan<- string, cgroup Cgroup, lastSample map[string]IoSample) (map[string]IoSample) { - blkio_io_service_bytes := FindStat(stderr, cgroup, "blkio", "blkio.io_service_bytes") - if blkio_io_service_bytes == "" { - return lastSample - } - - c, err := os.Open(blkio_io_service_bytes) +func DoBlkIoStats(cgroup Cgroup, lastSample map[string]IoSample) { + c, err := OpenStatFile(cgroup, "blkio", "blkio.io_service_bytes") if err != nil { - stderr <- fmt.Sprintf("crunchstat: open %s: %s", blkio_io_service_bytes, err) - return lastSample + return } defer c.Close() b := bufio.NewScanner(c) @@ -166,9 +163,6 @@ func DoBlkIoStats(stderr chan<- string, cgroup Cgroup, lastSample map[string]IoS } newSamples[device] = thisSample } - if lastSample == nil { - lastSample = make(map[string]IoSample) - } for dev, sample := range newSamples { if sample.txBytes < 0 || sample.rxBytes < 0 { continue @@ -177,13 +171,12 @@ func DoBlkIoStats(stderr chan<- string, cgroup Cgroup, lastSample map[string]IoS if prev, ok := lastSample[dev]; ok { delta = fmt.Sprintf(" -- interval %.4f seconds %d write %d read", sample.sampleTime.Sub(prev.sampleTime).Seconds(), - sample.txBytes - prev.txBytes, - sample.rxBytes - prev.rxBytes) + sample.txBytes-prev.txBytes, + sample.rxBytes-prev.rxBytes) } - stderr <- fmt.Sprintf("crunchstat: blkio:%s %d write %d read%s", dev, sample.txBytes, sample.rxBytes, delta) + LogPrintf("blkio:%s %d write %d read%s", dev, sample.txBytes, sample.rxBytes, delta) lastSample[dev] = sample } - return lastSample } type MemSample struct { @@ -191,14 +184,9 @@ type MemSample struct { memStat map[string]int64 } -func DoMemoryStats(stderr chan<- string, cgroup Cgroup) { - memory_stat := FindStat(stderr, cgroup, "memory", "memory.stat") - if memory_stat == "" { - return - } - c, err := os.Open(memory_stat) +func DoMemoryStats(cgroup Cgroup) { + c, err := OpenStatFile(cgroup, "memory", "memory.stat") if err != nil { - stderr <- fmt.Sprintf("crunchstat: open %s: %s", memory_stat, err) return } defer c.Close() @@ -219,44 +207,36 @@ func DoMemoryStats(stderr chan<- string, cgroup Cgroup) { outstat.WriteString(fmt.Sprintf(" %d %s", val, key)) } } - stderr <- fmt.Sprintf("crunchstat: mem%s", outstat.String()) + LogPrintf("mem%s", outstat.String()) } -func DoNetworkStats(stderr chan<- string, cgroup Cgroup, lastSample map[string]IoSample) (map[string]IoSample) { +func DoNetworkStats(cgroup Cgroup, lastSample map[string]IoSample) { sampleTime := time.Now() - stats, err := GetContainerNetStats(stderr, cgroup) - if err != nil { return lastSample } - - if lastSample == nil { - lastSample = make(map[string]IoSample) + stats, err := GetContainerNetStats(cgroup) + if err != nil { + return } + scanner := bufio.NewScanner(stats) - Iface: for scanner.Scan() { + for scanner.Scan() { var ifName string var rx, tx int64 - words := bufio.NewScanner(strings.NewReader(scanner.Text())) - words.Split(bufio.ScanWords) - wordIndex := 0 - for words.Scan() { - word := words.Text() - switch wordIndex { - case 0: - ifName = strings.TrimRight(word, ":") - case 1: - if _, err := fmt.Sscanf(word, "%d", &rx); err != nil { - continue Iface - } - case 9: - if _, err := fmt.Sscanf(word, "%d", &tx); err != nil { - continue Iface - } - } - wordIndex++ + words := strings.Fields(scanner.Text()) + if len(words) != 17 { + // Skip lines with wrong format + continue } - if ifName == "lo" || ifName == "" || wordIndex != 17 { + ifName = strings.TrimRight(words[0], ":") + if ifName == "lo" || ifName == "" { // Skip loopback interface and lines with wrong format continue } + if tx, err = strconv.ParseInt(words[9], 10, 64); err != nil { + continue + } + if rx, err = strconv.ParseInt(words[1], 10, 64); err != nil { + continue + } nextSample := IoSample{} nextSample.sampleTime = sampleTime nextSample.txBytes = tx @@ -266,58 +246,79 @@ func DoNetworkStats(stderr chan<- string, cgroup Cgroup, lastSample map[string]I interval := nextSample.sampleTime.Sub(lastSample.sampleTime).Seconds() delta = fmt.Sprintf(" -- interval %.4f seconds %d tx %d rx", interval, - tx - lastSample.txBytes, - rx - lastSample.rxBytes) + tx-lastSample.txBytes, + rx-lastSample.rxBytes) } - stderr <- fmt.Sprintf("crunchstat: net:%s %d tx %d rx%s", - ifName, tx, rx, delta) + LogPrintf("net:%s %d tx %d rx%s", ifName, tx, rx, delta) lastSample[ifName] = nextSample } - return lastSample } type CpuSample struct { + hasData bool // to distinguish the zero value from real data sampleTime time.Time user float64 sys float64 cpus int64 } -func DoCpuStats(stderr chan<- string, cgroup Cgroup, cpus int64, user_hz float64, lastSample *CpuSample) (*CpuSample) { - cpuacct_stat := FindStat(stderr, cgroup, "cpuacct", "cpuacct.stat") - if cpuacct_stat == "" { - return lastSample +// Return the number of CPUs available in the container. Return 0 if +// we can't figure out the real number of CPUs. +func GetCpuCount(cgroup Cgroup) int64 { + cpusetFile, err := OpenStatFile(cgroup, "cpuset", "cpuset.cpus") + if err != nil { + return 0 + } + defer cpusetFile.Close() + b, err := ReadAllOrWarn(cpusetFile) + sp := strings.Split(string(b), ",") + cpus := int64(0) + for _, v := range sp { + var min, max int64 + n, _ := fmt.Sscanf(v, "%d-%d", &min, &max) + if n == 2 { + cpus += (max - min) + 1 + } else { + cpus += 1 + } + } + return cpus +} + +func DoCpuStats(cgroup Cgroup, lastSample *CpuSample) { + statFile, err := OpenStatFile(cgroup, "cpuacct", "cpuacct.stat") + if err != nil { + return } - b, err := OpenAndReadAll(cpuacct_stat, stderr) + defer statFile.Close() + b, err := ReadAllOrWarn(statFile) if err != nil { - return lastSample + return } - nextSample := &CpuSample{time.Now(), 0, 0, cpus} + + nextSample := CpuSample{true, time.Now(), 0, 0, GetCpuCount(cgroup)} var userTicks, sysTicks int64 - fmt.Sscanf(string(b), "user %d\nsystem %d", &nextSample.user, &nextSample.sys) + fmt.Sscanf(string(b), "user %d\nsystem %d", &userTicks, &sysTicks) + user_hz := float64(C.sysconf(C._SC_CLK_TCK)) nextSample.user = float64(userTicks) / user_hz nextSample.sys = float64(sysTicks) / user_hz delta := "" - if lastSample != nil { + if lastSample.hasData { delta = fmt.Sprintf(" -- interval %.4f seconds %.4f user %.4f sys", nextSample.sampleTime.Sub(lastSample.sampleTime).Seconds(), - nextSample.user - lastSample.user, - nextSample.sys - lastSample.sys) + nextSample.user-lastSample.user, + nextSample.sys-lastSample.sys) } - stderr <- fmt.Sprintf("crunchstat: cpu %.4f user %.4f sys %d cpus%s", - nextSample.user, nextSample.sys, cpus, delta) - return nextSample + LogPrintf("cpu %.4f user %.4f sys %d cpus%s", + nextSample.user, nextSample.sys, nextSample.cpus, delta) + *lastSample = nextSample } -func PollCgroupStats(cgroup Cgroup, stderr chan string, poll int64, stop_poll_chan <-chan bool) { - var last_cpucount int64 = 0 - - user_hz := float64(C.sysconf(C._SC_CLK_TCK)) - - var lastNetSample map[string]IoSample = nil - var lastDiskSample map[string]IoSample = nil - var lastCpuSample *CpuSample = nil +func PollCgroupStats(cgroup Cgroup, poll int64, stop_poll_chan <-chan bool) { + var lastNetSample = map[string]IoSample{} + var lastDiskSample = map[string]IoSample{} + var lastCpuSample = CpuSample{} poll_chan := make(chan bool, 1) go func() { @@ -335,31 +336,10 @@ func PollCgroupStats(cgroup Cgroup, stderr chan string, poll int64, stop_poll_ch case <-poll_chan: // Emit stats, then select again. } - cpuset_cpus := FindStat(stderr, cgroup, "cpuset", "cpuset.cpus") - if cpuset_cpus != "" { - b, err := OpenAndReadAll(cpuset_cpus, stderr) - if err != nil { - // cgroup probably gone -- skip other stats too. - continue - } - sp := strings.Split(string(b), ",") - cpus := int64(0) - for _, v := range sp { - var min, max int64 - n, _ := fmt.Sscanf(v, "%d-%d", &min, &max) - if n == 2 { - cpus += (max - min) + 1 - } else { - cpus += 1 - } - } - last_cpucount = cpus - } - - DoMemoryStats(stderr, cgroup) - lastCpuSample = DoCpuStats(stderr, cgroup, last_cpucount, user_hz, lastCpuSample) - lastDiskSample = DoBlkIoStats(stderr, cgroup, lastDiskSample) - lastNetSample = DoNetworkStats(stderr, cgroup, lastNetSample) + DoMemoryStats(cgroup) + DoCpuStats(cgroup, &lastCpuSample) + DoBlkIoStats(cgroup, lastDiskSample) + DoNetworkStats(cgroup, lastNetSample) } } @@ -385,12 +365,12 @@ func run(logger *log.Logger) error { logger.Fatal("Must provide -cgroup-root") } - stderr_chan := make(chan string, 1) - defer close(stderr_chan) + logChan = make(chan string, 1) + defer close(logChan) finish_chan := make(chan bool) defer close(finish_chan) - go CopyChanToPipe(stderr_chan, os.Stderr) + go CopyChanToPipe(logChan, os.Stderr) var cmd *exec.Cmd @@ -422,7 +402,7 @@ func run(logger *log.Logger) error { if err != nil { logger.Fatal(err) } - go CopyPipeToChan(stderr_pipe, stderr_chan, finish_chan) + go CopyPipeToChan(stderr_pipe, logChan, finish_chan) // Run subprocess if err := cmd.Start(); err != nil { @@ -441,7 +421,7 @@ func run(logger *log.Logger) error { ok := false var i time.Duration for i = 0; i < time.Duration(wait)*time.Second; i += (100 * time.Millisecond) { - cid, err := OpenAndReadAll(cgroup_cidfile, nil) + cid, err := ioutil.ReadFile(cgroup_cidfile) if err == nil && len(cid) > 0 { ok = true container_id = string(cid) @@ -456,7 +436,7 @@ func run(logger *log.Logger) error { stop_poll_chan := make(chan bool, 1) cgroup := Cgroup{cgroup_root, cgroup_parent, container_id} - go PollCgroupStats(cgroup, stderr_chan, poll, stop_poll_chan) + go PollCgroupStats(cgroup, poll, stop_poll_chan) // When the child exits, tell the polling goroutine to stop. defer func() { stop_poll_chan <- true }()