services/fuse/arvados_fuse/crunchstat.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: AGPL-3.0
   4
   5 import sys
   6 import time
   7 from collections import namedtuple
   8
   9 Stat = namedtuple("Stat", ['name', 'get'])
  10
  11 class StatWriter(object):
  12     def __init__(self, prefix, interval, stats):
  13         self.prefix = prefix
  14         self.interval = interval
  15         self.stats = stats
  16         self.previous_stats = []
  17         self.update_previous_stats()
  18
  19     def update_previous_stats(self):
  20         self.previous_stats = [stat.get() for stat in self.stats]
  21
  22     def update(self):
  23         def append_by_type(string, name, value):
  24             if type(value) is float:
  25                 string += " %.6f %s" % (value, name)
  26             else:
  27                 string += " %s %s" % (str(value), name)
  28             return string
  29
  30         out = "crunchstat: %s" % self.prefix
  31         delta = "-- interval %.4f seconds" % self.interval
  32         for i, stat in enumerate(self.stats):
  33             value = stat.get()
  34             diff = value - self.previous_stats[i]
  35             delta = append_by_type(delta, stat.name, diff)
  36             out = append_by_type(out, stat.name, value)
  37
  38         sys.stderr.write("%s %s\n" % (out, delta))
  39         self.update_previous_stats()
  40
  41 def statlogger(interval, keep, ops):
  42     calls = StatWriter("keepcalls", interval, [
  43         Stat("put", keep.put_counter.get),
  44         Stat("get", keep.get_counter.get)
  45     ])
  46     net = StatWriter("net:keep0", interval, [
  47         Stat("tx", keep.upload_counter.get),
  48         Stat("rx", keep.download_counter.get)
  49     ])
  50     cache = StatWriter("keepcache", interval, [
  51         Stat("hit", keep.hits_counter.get),
  52         Stat("miss", keep.misses_counter.get)
  53     ])
  54     fuseops = StatWriter("fuseops", interval, [
  55         Stat("write", ops.write_ops_counter.get),
  56         Stat("read", ops.read_ops_counter.get)
  57     ])
  58     fusetimes = []
  59     for cur_op in ops.metric_op_names():
  60         name = "fuseop:{0}".format(cur_op)
  61         fusetimes.append(StatWriter(name, interval, [
  62             Stat("count", ops.metric_count_func(cur_op)),
  63             Stat("time", ops.metric_sum_func(cur_op))
  64         ]))
  65     blk = StatWriter("blkio:0:0", interval, [
  66         Stat("write", ops.write_counter.get),
  67         Stat("read", ops.read_counter.get)
  68     ])
  69
  70     while True:
  71         time.sleep(interval)
  72         calls.update()
  73         net.update()
  74         cache.update()
  75         blk.update()
  76         fuseops.update()
  77         for ftime in fusetimes:
  78             ftime.update()
  79
  80