6260: Expose queue sizes in /status.json. Fix sleep/race in trash_worker_test.

[arvados.git] / services / keepstore / handlers.go

diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 3898b55b61b7da393566930f5eeec742f24c2c85..a6665f61640dfdb5f0791b4fc16fc34f084fe4bf 100644 (file)
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -19,8 +19,9 @@ import (
        "net/http"
        "os"
        "regexp"
+       "runtime"
        "strconv"
-       "syscall"
+       "sync"
        "time"
 )
 
@@ -185,60 +186,74 @@ type VolumeStatus struct {
        BytesUsed  uint64 `json:"bytes_used"`
 }
 
+type PoolStatus struct {
+       Alloc uint64 `json:"BytesAllocated"`
+       Cap   int    `json:"BuffersMax"`
+       Len   int    `json:"BuffersInUse"`
+}
+
+type WorkQueueStatus struct {
+       InProgress  int
+       Outstanding int
+       Queued      int
+}
+
 type NodeStatus struct {
-       Volumes []*VolumeStatus `json:"volumes"`
+       Volumes    []*VolumeStatus `json:"volumes"`
+       BufferPool PoolStatus
+       PullQueue  WorkQueueStatus
+       TrashQueue WorkQueueStatus
+       Memory     runtime.MemStats
 }
 
+var st NodeStatus
+var stLock sync.Mutex
+
 func StatusHandler(resp http.ResponseWriter, req *http.Request) {
-       st := GetNodeStatus()
-       if jstat, err := json.Marshal(st); err == nil {
+       stLock.Lock()
+       readNodeStatus(&st)
+       jstat, err := json.Marshal(&st)
+       stLock.Unlock()
+       if err == nil {
                resp.Write(jstat)
        } else {
                log.Printf("json.Marshal: %s\n", err)
-               log.Printf("NodeStatus = %v\n", st)
+               log.Printf("NodeStatus = %v\n", &st)
                http.Error(resp, err.Error(), 500)
        }
 }
 
-// GetNodeStatus
-//     Returns a NodeStatus struct describing this Keep
-//     node's current status.
-//
-func GetNodeStatus() *NodeStatus {
-       st := new(NodeStatus)
-
-       st.Volumes = make([]*VolumeStatus, len(KeepVM.AllReadable()))
-       for i, vol := range KeepVM.AllReadable() {
-               st.Volumes[i] = vol.Status()
+// populate the given NodeStatus struct with current values.
+func readNodeStatus(st *NodeStatus) {
+       vols := KeepVM.AllReadable()
+       if cap(st.Volumes) < len(vols) {
+               st.Volumes = make([]*VolumeStatus, len(vols))
+       }
+       st.Volumes = st.Volumes[:0]
+       for _, vol := range vols {
+               if s := vol.Status(); s != nil {
+                       st.Volumes = append(st.Volumes, s)
+               }
        }
-       return st
+       st.BufferPool.Alloc = bufs.Alloc()
+       st.BufferPool.Cap = bufs.Cap()
+       st.BufferPool.Len = bufs.Len()
+       readWorkQueueStatus(&st.PullQueue, pullq)
+       readWorkQueueStatus(&st.TrashQueue, trashq)
+       runtime.ReadMemStats(&st.Memory)
 }
 
-// GetVolumeStatus
-//     Returns a VolumeStatus describing the requested volume.
-//
-func GetVolumeStatus(volume string) *VolumeStatus {
-       var fs syscall.Statfs_t
-       var devnum uint64
-
-       if fi, err := os.Stat(volume); err == nil {
-               devnum = fi.Sys().(*syscall.Stat_t).Dev
-       } else {
-               log.Printf("GetVolumeStatus: os.Stat: %s\n", err)
-               return nil
+// Populate a WorkQueueStatus. This is not atomic, so race conditions
+// can cause InProgress + Queued != Outstanding.
+func readWorkQueueStatus(st *WorkQueueStatus, q *WorkQueue) {
+       if q == nil {
+               // This should only happen during tests.
+               *st = WorkQueueStatus{}
+               return
        }
-
-       err := syscall.Statfs(volume, &fs)
-       if err != nil {
-               log.Printf("GetVolumeStatus: statfs: %s\n", err)
-               return nil
-       }
-       // These calculations match the way df calculates disk usage:
-       // "free" space is measured by fs.Bavail, but "used" space
-       // uses fs.Blocks - fs.Bfree.
-       free := fs.Bavail * uint64(fs.Bsize)
-       used := (fs.Blocks - fs.Bfree) * uint64(fs.Bsize)
-       return &VolumeStatus{volume, devnum, free, used}
+       st.InProgress = q.CountInProgress()
+       st.Outstanding = q.CountOutstanding()
+       st.Queued = q.CountQueued()
 }
 
 // DeleteHandler processes DELETE requests.