21121: Gets storage and compute usage

[arvados.git] / tools / cluster-activity / arvados_cluster_activity / main.py

diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index d0d643c4d406964d3a3f399ae750450488248da3..383f1b6a33df1541755520523fe6638b2ad44853 100755 (executable)
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -14,6 +14,7 @@ import csv
 import os
 from prometheus_api_client.utils import parse_datetime
 from datetime import timedelta
+import pandas
 
 from prometheus_api_client import PrometheusConnect, MetricsList, Metric
 
@@ -22,6 +23,7 @@ def parse_arguments(arguments):
     arg_parser.add_argument('--start', help='Start date for the report in YYYY-MM-DD format (UTC)')
     arg_parser.add_argument('--end', help='End date for the report in YYYY-MM-DD format (UTC)')
     arg_parser.add_argument('--days', type=int, help='Number of days before now() to start the report')
+    arg_parser.add_argument('--cluster', type=str, help='Cluster to query')
     args = arg_parser.parse_args(arguments)
 
     if args.days and (args.start or args.end):
@@ -61,50 +63,68 @@ def parse_arguments(arguments):
 
     return args, since, to
 
-def data_usage(prom, cluster):
-    metric_data = prom.get_current_metric_value(metric_name='arvados_keep_total_bytes', label_config={"cluster": cluster})
+def data_usage(prom, timestamp, cluster, label):
+    metric_data = prom.get_current_metric_value(metric_name='arvados_keep_total_bytes',
+                                                label_config={"cluster": cluster},
+                                                params={"time": timestamp.timestamp()})
 
     metric_object_list = MetricsList(metric_data)
-    #for item in metric_object_list:
-    #    print(item.metric_name, item.label_config, "\n")
 
     my_metric_object = metric_object_list[0] # one of the metrics from the list
 
-    #print(my_metric_object.metric_values)
     value = my_metric_object.metric_values.iloc[0]["y"]
 
     for scale in ["KiB", "MiB", "GiB", "TiB", "PiB"]:
         value = value / 1024
         if value < 1024:
-            print(value, scale)
+            v = "%.3f %s" % (value, scale)
+            print(label % v)
             break
 
 
-def container_usage(prom, cluster):
-
-    start_time = parse_datetime("7d")
-    end_time = parse_datetime("now")
+def container_usage(prom, start_time, end_time, metric, label, fn=None):
+    start = start_time
     chunk_size = timedelta(days=1)
+    cumulative = 0
 
-    metric_data = prom.get_metric_range_data(metric_name='arvados_dispatchcloud_containers_running',
-                                             label_config={"cluster": cluster},
-                                             start_time=start_time,
-                                             end_time=end_time,
-                                             chunk_size=chunk_size,
-                                             )
+    while start < end_time:
+        if start + chunk_size > end_time:
+            chunk_size = end_time - start
 
-    metric_object_list = MetricsList(metric_data)
-    my_metric_object = metric_object_list[0] # one of the metrics from the list
+        metric_data = prom.custom_query_range(metric,
+                                              start_time=start,
+                                              end_time=(start + chunk_size),
+                                              step=15
+                                              )
+
+        if "__name__" not in metric_data[0]["metric"]:
+            metric_data[0]["metric"]["__name__"] = metric
+
+        metric_object_list = MetricsList(metric_data)
+        my_metric_object = metric_object_list[0] # one of the metrics from the list
+
+        series = my_metric_object.metric_values.set_index(pandas.DatetimeIndex(my_metric_object.metric_values['ds']))
 
-    s = my_metric_object.metric_values.sum(numeric_only=True)
-    print(s["y"] / 4, "container minutes")
+        # Resample to 1 minute increments, fill in missing values
+        rs = series.resample("min").mean(1).ffill()
+
+        # Calculate the sum of values
+        #print(rs.sum()["y"])
+        cumulative += rs.sum()["y"]
+
+        start += chunk_size
+
+    if fn is not None:
+        cumulative = fn(cumulative)
+
+    print(label % cumulative)
 
 
 def main(arguments=None):
     if arguments is None:
         arguments = sys.argv[1:]
 
-    #args, since, to = parse_arguments(arguments)
+    args, since, to = parse_arguments(arguments)
 
     #arv = arvados.api()
 
@@ -113,11 +133,14 @@ def main(arguments=None):
 
     prom = PrometheusConnect(url=prom_host, headers={"Authorization": "Bearer "+prom_token})
 
-    for cluster in ("tordo", "pirca", "jutro"):
-        print(cluster)
-        data_usage(prom, cluster)
-        container_usage(prom, cluster)
-        print()
+    cluster = args.cluster
+
+    print(cluster, "between", since, "and", to, "timespan", (to-since))
+    data_usage(prom, since, cluster, "%s at start")
+    data_usage(prom, to - timedelta(minutes=240), cluster, "%s now")
+    container_usage(prom, since, to, "arvados_dispatchcloud_containers_running{cluster='%s'}" % cluster, '%.1f container hours', lambda x: x/60)
+    container_usage(prom, since, to, "sum(arvados_dispatchcloud_instances_price{cluster='%s'})" % cluster, '$%.2f dollars')
+    print()
 
 if __name__ == "__main__":
     main()