8409: Add recommendation if cache miss rate is above 0.5%. Fix tests.

diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index fcf94ef34f698b66900dc894e5c0e8c01155f8cf..6f8b74cc72caea8e412caab68751197f38116a14 100644 (file)
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -240,12 +240,14 @@ class Summarizer(object):
                  self.stats_max['net:keep0']['tx+rx__rate'],
                  lambda x: x / 1e6),
                 ('Keep cache miss rate {}%',
-                 float(self.job_tot['keepcache']['miss']) /
-                 float(self.job_tot['keepcalls']['get']),
+                 (float(self.job_tot['keepcache']['miss']) /
+                 float(self.job_tot['keepcalls']['get']))
+                 if self.job_tot['keepcalls']['get'] > 0 else 0,
                  lambda x: x * 100.0),
                 ('Keep cache utilization {}%',
-                 float(self.job_tot['blkio:0:0']['read']) /
-                 float(self.job_tot['net:keep0']['rx']),
+                 (float(self.job_tot['blkio:0:0']['read']) /
+                 float(self.job_tot['net:keep0']['rx']))
+                 if self.job_tot['net:keep0']['rx'] > 0 else 0,
                  lambda x: x * 100.0)):
             format_string, val, transform = args
             if val == float('-Inf'):
@@ -257,7 +259,8 @@ class Summarizer(object):
     def _recommend_gen(self):
         return itertools.chain(
             self._recommend_cpu(),
-            self._recommend_ram())
+            self._recommend_ram(),
+            self._recommend_keep_cache())
 
     def _recommend_cpu(self):
         """Recommend asking for 4 cores if max CPU usage was 333%"""
@@ -329,6 +332,23 @@ class Summarizer(object):
                 int(used_mib),
                 int(math.ceil(nearlygibs(used_mib))*AVAILABLE_RAM_RATIO*1024))
 
+    def _recommend_keep_cache(self):
+        """Recommend increasing keep cache if miss rate is above 0.5%"""
+        if self.job_tot['keepcalls']['get'] == 0:
+            return
+        miss_rate = float(self.job_tot['keepcache']['miss']) / float(self.job_tot['keepcalls']['get']) * 100.0
+        asked_mib = self.existing_constraints.get('keep_cache_mb_per_task', 256)
+
+        if miss_rate > 0.5:
+            yield (
+                '#!! {} Keep cache miss rate was {:.2f}% -- '
+                'try runtime_constraints "keep_cache_mb_per_task":{}'
+            ).format(
+                self.label,
+                miss_rate,
+                asked_mib*2)
+
+
     def _format(self, val):
         """Return a string representation of a stat.
 

diff --git a/tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report b/tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report
index e531e4b475f6aec5f7f4ce300db2b056f5db70a6..f0a60957bba706b18b6d41288f5fc46ded754b1c 100644 (file)
--- a/tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report
+++ b/tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report
@@ -29,5 +29,7 @@ time  elapsed 80      -       80
 # Max memory used by a single task: 0.35GB
 # Max network traffic in a single task: 1.79GB
 # Max network speed in a single interval: 42.58MB/s
+# Keep cache miss rate 0.00%
+# Keep cache utilization 0.00%
 #!! 4xphq-8i9sb-jq0ekny1xou3zoh max CPU usage was 13% -- try runtime_constraints "min_cores_per_node":1
 #!! 4xphq-8i9sb-jq0ekny1xou3zoh max RSS was 334 MiB -- try runtime_constraints "min_ram_mb_per_node":972

diff --git a/tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report b/tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report
index ac7c0a785adb409ffdcadf182b86bf760bfa7438..f9a34cfb98c7c6d42a633166b946b7c49db77cc1 100644 (file)
--- a/tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report
+++ b/tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report
@@ -18,5 +18,7 @@ time  elapsed 2       -       4
 # Max memory used by a single task: 0.00GB
 # Max network traffic in a single task: 0.00GB
 # Max network speed in a single interval: 0.00MB/s
+# Keep cache miss rate 0.00%
+# Keep cache utilization 0.00%
 #!! 4xphq-8i9sb-zvb2ocfycpomrup max CPU usage was 0% -- try runtime_constraints "min_cores_per_node":1
 #!! 4xphq-8i9sb-zvb2ocfycpomrup max RSS was 1 MiB -- try runtime_constraints "min_ram_mb_per_node":972

diff --git a/tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report b/tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report
index 6b4240cba8263065d28a00c536fcaaac44420d21..c54102d78a25d1158ba563a22de7279b3d39c2f2 100644 (file)
--- a/tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report
+++ b/tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report
@@ -18,5 +18,7 @@ time  elapsed 2       -       3
 # Max memory used by a single task: 0.00GB
 # Max network traffic in a single task: 0.00GB
 # Max network speed in a single interval: 0.00MB/s
+# Keep cache miss rate 0.00%
+# Keep cache utilization 0.00%
 #!! 4xphq-8i9sb-v831jm2uq0g2g9x max CPU usage was 0% -- try runtime_constraints "min_cores_per_node":1
 #!! 4xphq-8i9sb-v831jm2uq0g2g9x max RSS was 1 MiB -- try runtime_constraints "min_ram_mb_per_node":972