From dafb4b607a412111c29cfa7397334a9f18731b7a Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 2 Jul 2024 15:33:58 -0400 Subject: [PATCH] 21121: Add missing file and cluster-activity cwl Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- .../arvados_cluster_activity/prometheus.py | 69 +++++++++++++++++++ tools/cluster-activity/cluster-activity.cwl | 44 ++++++++++++ tools/cluster-activity/docker/Dockerfile | 2 +- tools/cluster-activity/make_docker.sh | 2 +- 4 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 tools/cluster-activity/arvados_cluster_activity/prometheus.py create mode 100644 tools/cluster-activity/cluster-activity.cwl diff --git a/tools/cluster-activity/arvados_cluster_activity/prometheus.py b/tools/cluster-activity/arvados_cluster_activity/prometheus.py new file mode 100644 index 0000000000..47e07d15a7 --- /dev/null +++ b/tools/cluster-activity/arvados_cluster_activity/prometheus.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +from datetime import timedelta, timezone + +def get_metric_usage(prom, start_time, end_time, metric, resampleTo="min"): + from prometheus_api_client.utils import parse_datetime + from prometheus_api_client import PrometheusConnect, MetricsList, Metric + import pandas + + start = start_time + chunk_size = timedelta(days=1) + + while start < end_time: + if start + chunk_size > end_time: + chunk_size = end_time - start + + metric_data = prom.custom_query_range(metric, + start_time=start, + end_time=(start + chunk_size), + step=15 + ) + + start += chunk_size + + if len(metric_data) == 0: + continue + + if "__name__" not in metric_data[0]["metric"]: + metric_data[0]["metric"]["__name__"] = metric + + metric_object_list = MetricsList(metric_data) + my_metric_object = metric_object_list[0] # one of the metrics from the list + + series = my_metric_object.metric_values.set_index(pandas.DatetimeIndex(my_metric_object.metric_values['ds'])) + + # Resample to 1 minute increments, fill in missing values + rs = series.resample(resampleTo).max(1).ffill() + + yield rs + +def get_data_usage(prom, timestamp, cluster): + from prometheus_api_client import PrometheusConnect, MetricsList, Metric + + metric_data = prom.get_current_metric_value(metric_name='arvados_keep_total_bytes', + label_config={"cluster": cluster}, + params={"time": timestamp.timestamp()}) + + metric_object_list = MetricsList(metric_data) + + if len(metric_data) == 0: + return + + my_metric_object = metric_object_list[0] # one of the metrics from the list + value = my_metric_object.metric_values.iloc[0]["y"] + + metric_data = prom.get_current_metric_value(metric_name='arvados_keep_dedup_byte_ratio', + label_config={"cluster": cluster}, + params={"time": timestamp.timestamp()}) + + if len(metric_data) == 0: + return (None, None) + + my_metric_object = MetricsList(metric_data)[0] + dedup_ratio = my_metric_object.metric_values.iloc[0]["y"] + + return value, dedup_ratio diff --git a/tools/cluster-activity/cluster-activity.cwl b/tools/cluster-activity/cluster-activity.cwl new file mode 100644 index 0000000000..bf20cfb6b6 --- /dev/null +++ b/tools/cluster-activity/cluster-activity.cwl @@ -0,0 +1,44 @@ +#!/usr/bin/env cwl-runner +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +cwlVersion: v1.2 +class: CommandLineTool +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" + +inputs: + reporting_days: int + prometheus_host: string + prometheus_apikey: string + +requirements: + DockerRequirement: + dockerPull: 'arvados/cluster-activity:2.8.0.dev20240702194009' + + InitialWorkDirRequirement: + listing: + - entryname: prometheus.env + entry: | + PROMETHEUS_HOST=$(inputs.prometheus_host) + PROMETHEUS_APIKEY=$(inputs.prometheus_apikey) + + arv:APIRequirement: {} + +hints: + cwltool:Secrets: + secrets: [prometheus_apikey] + +arguments: + - arv-cluster-activity + - {prefix: '--prometheus-auth', valueFrom: prometheus.env} + - {prefix: '--days', valueFrom: $(inputs.reporting_days)} + - {prefix: '--html-report-file', valueFrom: report.html} + +outputs: + report: + type: File + outputBinding: + glob: report.html diff --git a/tools/cluster-activity/docker/Dockerfile b/tools/cluster-activity/docker/Dockerfile index 5a8f435df9..977c7e085a 100644 --- a/tools/cluster-activity/docker/Dockerfile +++ b/tools/cluster-activity/docker/Dockerfile @@ -10,5 +10,5 @@ RUN python3 -mvenv /usr/local/cluster-activity && \ /usr/local/cluster-activity/bin/pip install \ $(ls /root/arvados-python-client-*.tar.gz) \ $(ls /root/crunchstat_summary-*.tar.gz) \ - $(ls /root/arvados-user-activity-*.tar.gz)\[prometheus\] && \ + $(ls /root/arvados-cluster-activity-*.tar.gz)\[prometheus\] && \ ln -s /usr/local/cluster-activity/bin/arv-cluster-activity /usr/local/bin diff --git a/tools/cluster-activity/make_docker.sh b/tools/cluster-activity/make_docker.sh index 596f686979..c57975eaf5 100755 --- a/tools/cluster-activity/make_docker.sh +++ b/tools/cluster-activity/make_docker.sh @@ -19,7 +19,7 @@ VERSION=$(python3 arvados_version.py) rm -f docker/*.tar.gz cp ../../sdk/python/dist/arvados-python-client-${SDK_VERSION}.tar.gz \ ../crunchstat-summary/dist/crunchstat_summary-${CRUNCHSTAT_VERSION}.tar.gz \ - dist/arvados-user-activity-${VERSION}.tar.gz \ + dist/arvados-cluster-activity-${VERSION}.tar.gz \ docker/ cd docker docker build -t arvados/cluster-activity:$VERSION . -- 2.30.2