services/keepstore/metrics.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "fmt"
   9
  10         "git.curoverse.com/arvados.git/sdk/go/httpserver"
  11         "github.com/prometheus/client_golang/prometheus"
  12 )
  13
  14 type nodeMetrics struct {
  15         reg *prometheus.Registry
  16         rc  httpserver.RequestCounter
  17 }
  18
  19 func (m *nodeMetrics) setup() {
  20         m.reg.MustRegister(prometheus.NewGaugeFunc(
  21                 prometheus.GaugeOpts{
  22                         Namespace: "arvados",
  23                         Subsystem: "keepstore",
  24                         Name:      "bufferpool_bytes_allocated",
  25                         Help:      "Number of bytes allocated to buffers",
  26                 },
  27                 func() float64 { return float64(bufs.Alloc()) },
  28         ))
  29         m.reg.MustRegister(prometheus.NewGaugeFunc(
  30                 prometheus.GaugeOpts{
  31                         Namespace: "arvados",
  32                         Subsystem: "keepstore",
  33                         Name:      "bufferpool_buffers_max",
  34                         Help:      "Maximum number of buffers allowed",
  35                 },
  36                 func() float64 { return float64(bufs.Cap()) },
  37         ))
  38         m.reg.MustRegister(prometheus.NewGaugeFunc(
  39                 prometheus.GaugeOpts{
  40                         Namespace: "arvados",
  41                         Subsystem: "keepstore",
  42                         Name:      "bufferpool_buffers_in_use",
  43                         Help:      "Number of buffers in use",
  44                 },
  45                 func() float64 { return float64(bufs.Len()) },
  46         ))
  47         m.reg.MustRegister(prometheus.NewGaugeFunc(
  48                 prometheus.GaugeOpts{
  49                         Namespace: "arvados",
  50                         Subsystem: "keepstore",
  51                         Name:      "pull_queue_in_progress",
  52                         Help:      "Number of pull requests in progress",
  53                 },
  54                 func() float64 { return float64(getWorkQueueStatus(pullq).InProgress) },
  55         ))
  56         m.reg.MustRegister(prometheus.NewGaugeFunc(
  57                 prometheus.GaugeOpts{
  58                         Namespace: "arvados",
  59                         Subsystem: "keepstore",
  60                         Name:      "pull_queue_queued",
  61                         Help:      "Number of queued pull requests",
  62                 },
  63                 func() float64 { return float64(getWorkQueueStatus(pullq).Queued) },
  64         ))
  65         m.reg.MustRegister(prometheus.NewGaugeFunc(
  66                 prometheus.GaugeOpts{
  67                         Namespace: "arvados",
  68                         Subsystem: "keepstore",
  69                         Name:      "trash_queue_in_progress",
  70                         Help:      "Number of trash requests in progress",
  71                 },
  72                 func() float64 { return float64(getWorkQueueStatus(trashq).InProgress) },
  73         ))
  74         m.reg.MustRegister(prometheus.NewGaugeFunc(
  75                 prometheus.GaugeOpts{
  76                         Namespace: "arvados",
  77                         Subsystem: "keepstore",
  78                         Name:      "trash_queue_queued",
  79                         Help:      "Number of queued trash requests",
  80                 },
  81                 func() float64 { return float64(getWorkQueueStatus(trashq).Queued) },
  82         ))
  83         m.reg.MustRegister(prometheus.NewGaugeFunc(
  84                 prometheus.GaugeOpts{
  85                         Namespace: "arvados",
  86                         Subsystem: "keepstore",
  87                         Name:      "requests_current",
  88                         Help:      "Number of requests in progress",
  89                 },
  90                 func() float64 { return float64(m.rc.Current()) },
  91         ))
  92         m.reg.MustRegister(prometheus.NewGaugeFunc(
  93                 prometheus.GaugeOpts{
  94                         Namespace: "arvados",
  95                         Subsystem: "keepstore",
  96                         Name:      "requests_max",
  97                         Help:      "Maximum number of concurrent requests",
  98                 },
  99                 func() float64 { return float64(m.rc.Max()) },
 100         ))
 101         // Register individual volume's metrics
 102         vols := KeepVM.AllReadable()
 103         for _, vol := range vols {
 104                 labels := prometheus.Labels{
 105                         "label":         vol.String(),
 106                         "mount_point":   vol.Status().MountPoint,
 107                         "device_number": fmt.Sprintf("%d", vol.Status().DeviceNum),
 108                 }
 109                 if vol, ok := vol.(InternalMetricser); ok {
 110                         // Per-driver internal metrics
 111                         vol.SetupInternalMetrics(m.reg, labels)
 112                 }
 113                 m.reg.Register(prometheus.NewGaugeFunc(
 114                         prometheus.GaugeOpts{
 115                                 Namespace:   "arvados",
 116                                 Subsystem:   "keepstore",
 117                                 Name:        "volume_bytes_free",
 118                                 Help:        "Number of free bytes on the volume",
 119                                 ConstLabels: labels,
 120                         },
 121                         func() float64 { return float64(vol.Status().BytesFree) },
 122                 ))
 123                 m.reg.Register(prometheus.NewGaugeFunc(
 124                         prometheus.GaugeOpts{
 125                                 Namespace:   "arvados",
 126                                 Subsystem:   "keepstore",
 127                                 Name:        "volume_bytes_used",
 128                                 Help:        "Number of used bytes on the volume",
 129                                 ConstLabels: labels,
 130                         },
 131                         func() float64 { return float64(vol.Status().BytesUsed) },
 132                 ))
 133                 m.reg.Register(prometheus.NewGaugeFunc(
 134                         prometheus.GaugeOpts{
 135                                 Namespace:   "arvados",
 136                                 Subsystem:   "keepstore",
 137                                 Name:        "volume_io_errors",
 138                                 Help:        "Number of I/O errors",
 139                                 ConstLabels: labels,
 140                         },
 141                         func() float64 { return float64(KeepVM.VolumeStats(vol).Errors) },
 142                 ))
 143                 m.reg.Register(prometheus.NewGaugeFunc(
 144                         prometheus.GaugeOpts{
 145                                 Namespace:   "arvados",
 146                                 Subsystem:   "keepstore",
 147                                 Name:        "volume_io_ops",
 148                                 Help:        "Number of I/O operations",
 149                                 ConstLabels: labels,
 150                         },
 151                         func() float64 { return float64(KeepVM.VolumeStats(vol).Ops) },
 152                 ))
 153                 m.reg.Register(prometheus.NewGaugeFunc(
 154                         prometheus.GaugeOpts{
 155                                 Namespace:   "arvados",
 156                                 Subsystem:   "keepstore",
 157                                 Name:        "volume_io_compare_ops",
 158                                 Help:        "Number of I/O compare operations",
 159                                 ConstLabels: labels,
 160                         },
 161                         func() float64 { return float64(KeepVM.VolumeStats(vol).CompareOps) },
 162                 ))
 163                 m.reg.Register(prometheus.NewGaugeFunc(
 164                         prometheus.GaugeOpts{
 165                                 Namespace:   "arvados",
 166                                 Subsystem:   "keepstore",
 167                                 Name:        "volume_io_get_ops",
 168                                 Help:        "Number of I/O get operations",
 169                                 ConstLabels: labels,
 170                         },
 171                         func() float64 { return float64(KeepVM.VolumeStats(vol).GetOps) },
 172                 ))
 173                 m.reg.Register(prometheus.NewGaugeFunc(
 174                         prometheus.GaugeOpts{
 175                                 Namespace:   "arvados",
 176                                 Subsystem:   "keepstore",
 177                                 Name:        "volume_io_put_ops",
 178                                 Help:        "Number of I/O put operations",
 179                                 ConstLabels: labels,
 180                         },
 181                         func() float64 { return float64(KeepVM.VolumeStats(vol).PutOps) },
 182                 ))
 183                 m.reg.Register(prometheus.NewGaugeFunc(
 184                         prometheus.GaugeOpts{
 185                                 Namespace:   "arvados",
 186                                 Subsystem:   "keepstore",
 187                                 Name:        "volume_io_touch_ops",
 188                                 Help:        "Number of I/O touch operations",
 189                                 ConstLabels: labels,
 190                         },
 191                         func() float64 { return float64(KeepVM.VolumeStats(vol).TouchOps) },
 192                 ))
 193                 m.reg.Register(prometheus.NewGaugeFunc(
 194                         prometheus.GaugeOpts{
 195                                 Namespace:   "arvados",
 196                                 Subsystem:   "keepstore",
 197                                 Name:        "volume_io_input_bytes",
 198                                 Help:        "Number of input bytes",
 199                                 ConstLabels: labels,
 200                         },
 201                         func() float64 { return float64(KeepVM.VolumeStats(vol).InBytes) },
 202                 ))
 203                 m.reg.Register(prometheus.NewGaugeFunc(
 204                         prometheus.GaugeOpts{
 205                                 Namespace:   "arvados",
 206                                 Subsystem:   "keepstore",
 207                                 Name:        "volume_io_output_bytes",
 208                                 Help:        "Number of output bytes",
 209                                 ConstLabels: labels,
 210                         },
 211                         func() float64 { return float64(KeepVM.VolumeStats(vol).OutBytes) },
 212                 ))
 213         }
 214 }