1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
10 "git.curoverse.com/arvados.git/sdk/go/httpserver"
11 "github.com/prometheus/client_golang/prometheus"
14 type nodeMetrics struct {
15 reg *prometheus.Registry
18 func (m *nodeMetrics) setupBufferPoolMetrics(b *bufferPool) {
19 m.reg.MustRegister(prometheus.NewGaugeFunc(
22 Subsystem: "keepstore",
23 Name: "bufferpool_bytes_allocated",
24 Help: "Number of bytes allocated to buffers",
26 func() float64 { return float64(b.Alloc()) },
28 m.reg.MustRegister(prometheus.NewGaugeFunc(
31 Subsystem: "keepstore",
32 Name: "bufferpool_buffers_max",
33 Help: "Maximum number of buffers allowed",
35 func() float64 { return float64(b.Cap()) },
37 m.reg.MustRegister(prometheus.NewGaugeFunc(
40 Subsystem: "keepstore",
41 Name: "bufferpool_buffers_in_use",
42 Help: "Number of buffers in use",
44 func() float64 { return float64(b.Len()) },
48 func (m *nodeMetrics) setupWorkQueueMetrics(q *WorkQueue, qName string) {
49 m.reg.MustRegister(prometheus.NewGaugeFunc(
52 Subsystem: "keepstore",
53 Name: fmt.Sprintf("%s_queue_in_progress", qName),
54 Help: fmt.Sprintf("Number of %s requests in progress", qName),
56 func() float64 { return float64(getWorkQueueStatus(q).InProgress) },
58 m.reg.MustRegister(prometheus.NewGaugeFunc(
61 Subsystem: "keepstore",
62 Name: fmt.Sprintf("%s_queue_queued", qName),
63 Help: fmt.Sprintf("Number of queued %s requests", qName),
65 func() float64 { return float64(getWorkQueueStatus(q).Queued) },
69 func (m *nodeMetrics) setupRequestMetrics(rc httpserver.RequestCounter) {
70 m.reg.MustRegister(prometheus.NewGaugeFunc(
73 Subsystem: "keepstore",
74 Name: "requests_current",
75 Help: "Number of requests in progress",
77 func() float64 { return float64(rc.Current()) },
79 m.reg.MustRegister(prometheus.NewGaugeFunc(
82 Subsystem: "keepstore",
84 Help: "Maximum number of concurrent requests",
86 func() float64 { return float64(rc.Max()) },
90 type volumeMetricsVecs struct {
91 reg *prometheus.Registry
92 BytesFree *prometheus.GaugeVec
93 BytesUsed *prometheus.GaugeVec
94 Errors *prometheus.CounterVec
95 Ops *prometheus.CounterVec
96 CompareOps *prometheus.CounterVec
97 GetOps *prometheus.CounterVec
98 PutOps *prometheus.CounterVec
99 TouchOps *prometheus.CounterVec
100 InBytes *prometheus.CounterVec
101 OutBytes *prometheus.CounterVec
102 ErrorCodes *prometheus.CounterVec
105 type volumeMetrics struct {
106 reg *prometheus.Registry
108 internalCounters map[string]*prometheus.CounterVec
109 BytesFree prometheus.Gauge
110 BytesUsed prometheus.Gauge
111 Errors prometheus.Counter
112 Ops prometheus.Counter
113 CompareOps prometheus.Counter
114 GetOps prometheus.Counter
115 PutOps prometheus.Counter
116 TouchOps prometheus.Counter
117 InBytes prometheus.Counter
118 OutBytes prometheus.Counter
119 ErrorCodes *prometheus.CounterVec
122 func newVolumeMetricsVecs(reg *prometheus.Registry) *volumeMetricsVecs {
123 m := &volumeMetricsVecs{
126 m.BytesFree = prometheus.NewGaugeVec(
127 prometheus.GaugeOpts{
128 Namespace: "arvados",
129 Subsystem: "keepstore",
130 Name: "volume_bytes_free",
131 Help: "Number of free bytes on the volume",
133 []string{"label", "mount_point", "device_number"},
135 reg.MustRegister(m.BytesFree)
136 m.BytesUsed = prometheus.NewGaugeVec(
137 prometheus.GaugeOpts{
138 Namespace: "arvados",
139 Subsystem: "keepstore",
140 Name: "volume_bytes_used",
141 Help: "Number of used bytes on the volume",
143 []string{"label", "mount_point", "device_number"},
145 reg.MustRegister(m.BytesUsed)
146 m.Errors = prometheus.NewCounterVec(
147 prometheus.CounterOpts{
148 Namespace: "arvados",
149 Subsystem: "keepstore",
150 Name: "volume_io_errors",
151 Help: "Number of volume I/O errors",
153 []string{"label", "mount_point", "device_number"},
155 reg.MustRegister(m.Errors)
156 m.Ops = prometheus.NewCounterVec(
157 prometheus.CounterOpts{
158 Namespace: "arvados",
159 Subsystem: "keepstore",
160 Name: "volume_io_ops",
161 Help: "Number of volume I/O operations",
163 []string{"label", "mount_point", "device_number"},
165 reg.MustRegister(m.Ops)
166 m.CompareOps = prometheus.NewCounterVec(
167 prometheus.CounterOpts{
168 Namespace: "arvados",
169 Subsystem: "keepstore",
170 Name: "volume_io_compare_ops",
171 Help: "Number of volume I/O compare operations",
173 []string{"label", "mount_point", "device_number"},
175 reg.MustRegister(m.CompareOps)
176 m.GetOps = prometheus.NewCounterVec(
177 prometheus.CounterOpts{
178 Namespace: "arvados",
179 Subsystem: "keepstore",
180 Name: "volume_io_get_ops",
181 Help: "Number of volume I/O get operations",
183 []string{"label", "mount_point", "device_number"},
185 reg.MustRegister(m.GetOps)
186 m.PutOps = prometheus.NewCounterVec(
187 prometheus.CounterOpts{
188 Namespace: "arvados",
189 Subsystem: "keepstore",
190 Name: "volume_io_put_ops",
191 Help: "Number of volume I/O put operations",
193 []string{"label", "mount_point", "device_number"},
195 reg.MustRegister(m.PutOps)
196 m.TouchOps = prometheus.NewCounterVec(
197 prometheus.CounterOpts{
198 Namespace: "arvados",
199 Subsystem: "keepstore",
200 Name: "volume_io_touch_ops",
201 Help: "Number of volume I/O touch operations",
203 []string{"label", "mount_point", "device_number"},
205 reg.MustRegister(m.TouchOps)
206 m.InBytes = prometheus.NewCounterVec(
207 prometheus.CounterOpts{
208 Namespace: "arvados",
209 Subsystem: "keepstore",
210 Name: "volume_io_in_bytes",
211 Help: "Number of input bytes",
213 []string{"label", "mount_point", "device_number"},
215 reg.MustRegister(m.InBytes)
216 m.OutBytes = prometheus.NewCounterVec(
217 prometheus.CounterOpts{
218 Namespace: "arvados",
219 Subsystem: "keepstore",
220 Name: "volume_io_out_bytes",
221 Help: "Number of output bytes",
223 []string{"label", "mount_point", "device_number"},
225 reg.MustRegister(m.OutBytes)
226 m.ErrorCodes = prometheus.NewCounterVec(
227 prometheus.CounterOpts{
228 Namespace: "arvados",
229 Subsystem: "keepstore",
230 Name: "volume_io_error_codes",
231 Help: "Number of I/O errors by error code",
233 []string{"label", "mount_point", "device_number", "error_code"},
235 reg.MustRegister(m.ErrorCodes)
240 func (m *volumeMetricsVecs) curryWith(lbl string, mnt string, dev string) *volumeMetrics {
241 lbls := []string{lbl, mnt, dev}
242 curried := &volumeMetrics{
245 internalCounters: make(map[string]*prometheus.CounterVec),
246 BytesFree: m.BytesFree.WithLabelValues(lbls...),
247 BytesUsed: m.BytesUsed.WithLabelValues(lbls...),
248 Errors: m.Errors.WithLabelValues(lbls...),
249 Ops: m.Ops.WithLabelValues(lbls...),
250 CompareOps: m.CompareOps.WithLabelValues(lbls...),
251 GetOps: m.GetOps.WithLabelValues(lbls...),
252 PutOps: m.PutOps.WithLabelValues(lbls...),
253 TouchOps: m.TouchOps.WithLabelValues(lbls...),
254 InBytes: m.InBytes.WithLabelValues(lbls...),
255 OutBytes: m.OutBytes.WithLabelValues(lbls...),
256 ErrorCodes: m.ErrorCodes.MustCurryWith(prometheus.Labels{
259 "device_number": dev,
265 // Returns a driver specific counter, creating it when needed. The 'name' argument
266 // should include the driver prefix.
267 func (m *volumeMetrics) getInternalCounter(name string, help string) prometheus.Counter {
268 counterVec, ok := m.internalCounters[name]
270 counterVec = prometheus.NewCounterVec(
271 prometheus.CounterOpts{
272 Namespace: "arvados",
273 Subsystem: "keepstore",
277 []string{"label", "mount_point", "device_number"},
279 m.reg.MustRegister(counterVec)
280 m.internalCounters[name] = counterVec
282 return counterVec.WithLabelValues(m.lbls...)