X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/64efd1030538d59821ce288a7674e29d49c35744..6f18406060f2ac32d505db14ceab97b08431ce04:/services/keepstore/volume_generic_test.go

diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go
index 1738fe9b51..d5a413693f 100644
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@ -1,7 +1,12 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
 	"fmt"
 	"os"
@@ -13,6 +18,8 @@ import (
 
 	"git.curoverse.com/arvados.git/sdk/go/arvados"
 	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
 )
 
 type TB interface {
@@ -70,6 +77,8 @@ func DoGenericVolumeTests(t TB, factory TestableVolumeFactory) {
 
 	testStatus(t, factory)
 
+	testMetrics(t, factory)
+
 	testString(t, factory)
 
 	testUpdateReadOnly(t, factory)
@@ -92,7 +101,7 @@ func testGet(t TB, factory TestableVolumeFactory) {
 	v.PutRaw(TestHash, TestBlock)
 
 	buf := make([]byte, BlockSize)
-	n, err := v.Get(TestHash, buf)
+	n, err := v.Get(context.Background(), TestHash, buf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -109,7 +118,7 @@ func testGetNoSuchBlock(t TB, factory TestableVolumeFactory) {
 	defer v.Teardown()
 
 	buf := make([]byte, BlockSize)
-	if _, err := v.Get(TestHash2, buf); err == nil {
+	if _, err := v.Get(context.Background(), TestHash2, buf); err == nil {
 		t.Errorf("Expected error while getting non-existing block %v", TestHash2)
 	}
 }
@@ -121,7 +130,7 @@ func testCompareNonexistent(t TB, factory TestableVolumeFactory) {
 	v := factory(t)
 	defer v.Teardown()
 
-	err := v.Compare(TestHash, TestBlock)
+	err := v.Compare(context.Background(), TestHash, TestBlock)
 	if err != os.ErrNotExist {
 		t.Errorf("Got err %T %q, expected os.ErrNotExist", err, err)
 	}
@@ -136,7 +145,7 @@ func testCompareSameContent(t TB, factory TestableVolumeFactory, testHash string
 	v.PutRaw(testHash, testData)
 
 	// Compare the block locator with same content
-	err := v.Compare(testHash, testData)
+	err := v.Compare(context.Background(), testHash, testData)
 	if err != nil {
 		t.Errorf("Got err %q, expected nil", err)
 	}
@@ -154,7 +163,7 @@ func testCompareWithCollision(t TB, factory TestableVolumeFactory, testHash stri
 	v.PutRaw(testHash, testDataA)
 
 	// Compare the block locator with different content; collision
-	err := v.Compare(TestHash, testDataB)
+	err := v.Compare(context.Background(), TestHash, testDataB)
 	if err == nil {
 		t.Errorf("Got err nil, expected error due to collision")
 	}
@@ -170,7 +179,7 @@ func testCompareWithCorruptStoredData(t TB, factory TestableVolumeFactory, testH
 
 	v.PutRaw(TestHash, testDataB)
 
-	err := v.Compare(testHash, testDataA)
+	err := v.Compare(context.Background(), testHash, testDataA)
 	if err == nil || err == CollisionError {
 		t.Errorf("Got err %+v, expected non-collision error", err)
 	}
@@ -186,12 +195,12 @@ func testPutBlockWithSameContent(t TB, factory TestableVolumeFactory, testHash s
 		return
 	}
 
-	err := v.Put(testHash, testData)
+	err := v.Put(context.Background(), testHash, testData)
 	if err != nil {
 		t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
 	}
 
-	err = v.Put(testHash, testData)
+	err = v.Put(context.Background(), testHash, testData)
 	if err != nil {
 		t.Errorf("Got err putting block second time %q: %q, expected nil", TestBlock, err)
 	}
@@ -209,9 +218,9 @@ func testPutBlockWithDifferentContent(t TB, factory TestableVolumeFactory, testH
 
 	v.PutRaw(testHash, testDataA)
 
-	putErr := v.Put(testHash, testDataB)
+	putErr := v.Put(context.Background(), testHash, testDataB)
 	buf := make([]byte, BlockSize)
-	n, getErr := v.Get(testHash, buf)
+	n, getErr := v.Get(context.Background(), testHash, buf)
 	if putErr == nil {
 		// Put must not return a nil error unless it has
 		// overwritten the existing data.
@@ -238,23 +247,23 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
 		return
 	}
 
-	err := v.Put(TestHash, TestBlock)
+	err := v.Put(context.Background(), TestHash, TestBlock)
 	if err != nil {
 		t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
 	}
 
-	err = v.Put(TestHash2, TestBlock2)
+	err = v.Put(context.Background(), TestHash2, TestBlock2)
 	if err != nil {
 		t.Errorf("Got err putting block %q: %q, expected nil", TestBlock2, err)
 	}
 
-	err = v.Put(TestHash3, TestBlock3)
+	err = v.Put(context.Background(), TestHash3, TestBlock3)
 	if err != nil {
 		t.Errorf("Got err putting block %q: %q, expected nil", TestBlock3, err)
 	}
 
 	data := make([]byte, BlockSize)
-	n, err := v.Get(TestHash, data)
+	n, err := v.Get(context.Background(), TestHash, data)
 	if err != nil {
 		t.Error(err)
 	} else {
@@ -263,7 +272,7 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
 		}
 	}
 
-	n, err = v.Get(TestHash2, data)
+	n, err = v.Get(context.Background(), TestHash2, data)
 	if err != nil {
 		t.Error(err)
 	} else {
@@ -272,7 +281,7 @@ func testPutMultipleBlocks(t TB, factory TestableVolumeFactory) {
 		}
 	}
 
-	n, err = v.Get(TestHash3, data)
+	n, err = v.Get(context.Background(), TestHash3, data)
 	if err != nil {
 		t.Error(err)
 	} else {
@@ -294,7 +303,7 @@ func testPutAndTouch(t TB, factory TestableVolumeFactory) {
 		return
 	}
 
-	if err := v.Put(TestHash, TestBlock); err != nil {
+	if err := v.Put(context.Background(), TestHash, TestBlock); err != nil {
 		t.Error(err)
 	}
 
@@ -314,7 +323,7 @@ func testPutAndTouch(t TB, factory TestableVolumeFactory) {
 	}
 
 	// Write the same block again.
-	if err := v.Put(TestHash, TestBlock); err != nil {
+	if err := v.Put(context.Background(), TestHash, TestBlock); err != nil {
 		t.Error(err)
 	}
 
@@ -437,13 +446,13 @@ func testDeleteNewBlock(t TB, factory TestableVolumeFactory) {
 		return
 	}
 
-	v.Put(TestHash, TestBlock)
+	v.Put(context.Background(), TestHash, TestBlock)
 
 	if err := v.Trash(TestHash); err != nil {
 		t.Error(err)
 	}
 	data := make([]byte, BlockSize)
-	n, err := v.Get(TestHash, data)
+	n, err := v.Get(context.Background(), TestHash, data)
 	if err != nil {
 		t.Error(err)
 	} else if bytes.Compare(data[:n], TestBlock) != 0 {
@@ -463,14 +472,14 @@ func testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
 		return
 	}
 
-	v.Put(TestHash, TestBlock)
+	v.Put(context.Background(), TestHash, TestBlock)
 	v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
 
 	if err := v.Trash(TestHash); err != nil {
 		t.Error(err)
 	}
 	data := make([]byte, BlockSize)
-	if _, err := v.Get(TestHash, data); err == nil || !os.IsNotExist(err) {
+	if _, err := v.Get(context.Background(), TestHash, data); err == nil || !os.IsNotExist(err) {
 		t.Errorf("os.IsNotExist(%v) should have been true", err)
 	}
 
@@ -479,7 +488,7 @@ func testDeleteOldBlock(t TB, factory TestableVolumeFactory) {
 		t.Fatalf("os.IsNotExist(%v) should have been true", err)
 	}
 
-	err = v.Compare(TestHash, TestBlock)
+	err = v.Compare(context.Background(), TestHash, TestBlock)
 	if err == nil || !os.IsNotExist(err) {
 		t.Fatalf("os.IsNotExist(%v) should have been true", err)
 	}
@@ -528,6 +537,84 @@ func testStatus(t TB, factory TestableVolumeFactory) {
 	}
 }
 
+func getValueFrom(cv *prometheus.CounterVec, lbls prometheus.Labels) float64 {
+	c, _ := cv.GetMetricWith(lbls)
+	pb := &dto.Metric{}
+	c.Write(pb)
+	return pb.GetCounter().GetValue()
+}
+
+func testMetrics(t TB, factory TestableVolumeFactory) {
+	var err error
+
+	v := factory(t)
+	defer v.Teardown()
+	reg := prometheus.NewRegistry()
+	vm := newVolumeMetricsVecs(reg)
+
+	err = v.Start(vm)
+	if err != nil {
+		t.Error("Failed Start(): ", err)
+	}
+	opsC, _, ioC := vm.getCounterVecsFor(prometheus.Labels{"device_id": v.DeviceID()})
+
+	if ioC == nil {
+		t.Error("ioBytes CounterVec is nil")
+		return
+	}
+
+	if getValueFrom(ioC, prometheus.Labels{"direction": "out"})+
+		getValueFrom(ioC, prometheus.Labels{"direction": "in"}) > 0 {
+		t.Error("ioBytes counter should be zero")
+	}
+
+	if opsC == nil {
+		t.Error("opsCounter CounterVec is nil")
+		return
+	}
+
+	var c, writeOpCounter, readOpCounter float64
+
+	readOpType, writeOpType := v.ReadWriteOperationLabelValues()
+	writeOpCounter = getValueFrom(opsC, prometheus.Labels{"operation": writeOpType})
+	readOpCounter = getValueFrom(opsC, prometheus.Labels{"operation": readOpType})
+
+	// Test Put if volume is writable
+	if v.Writable() {
+		err = v.Put(context.Background(), TestHash, TestBlock)
+		if err != nil {
+			t.Errorf("Got err putting block %q: %q, expected nil", TestBlock, err)
+		}
+		// Check that the write operations counter increased
+		c = getValueFrom(opsC, prometheus.Labels{"operation": writeOpType})
+		if c <= writeOpCounter {
+			t.Error("Operation(s) not counted on Put")
+		}
+		// Check that bytes counter is > 0
+		if getValueFrom(ioC, prometheus.Labels{"direction": "out"}) == 0 {
+			t.Error("ioBytes{direction=out} counter shouldn't be zero")
+		}
+	} else {
+		v.PutRaw(TestHash, TestBlock)
+	}
+
+	buf := make([]byte, BlockSize)
+	_, err = v.Get(context.Background(), TestHash, buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check that the operations counter increased
+	c = getValueFrom(opsC, prometheus.Labels{"operation": readOpType})
+	if c <= readOpCounter {
+		t.Error("Operation(s) not counted on Get")
+	}
+	// Check that the bytes "in" counter is > 0
+	if getValueFrom(ioC, prometheus.Labels{"direction": "in"}) == 0 {
+		t.Error("ioBytes{direction=in} counter shouldn't be zero")
+	}
+}
+
 // Invoke String for the volume; expect non-empty result
 // Test should pass for both writable and read-only volumes
 func testString(t TB, factory TestableVolumeFactory) {
@@ -553,17 +640,17 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
 	buf := make([]byte, BlockSize)
 
 	// Get from read-only volume should succeed
-	_, err := v.Get(TestHash, buf)
+	_, err := v.Get(context.Background(), TestHash, buf)
 	if err != nil {
 		t.Errorf("got err %v, expected nil", err)
 	}
 
 	// Put a new block to read-only volume should result in error
-	err = v.Put(TestHash2, TestBlock2)
+	err = v.Put(context.Background(), TestHash2, TestBlock2)
 	if err == nil {
 		t.Errorf("Expected error when putting block in a read-only volume")
 	}
-	_, err = v.Get(TestHash2, buf)
+	_, err = v.Get(context.Background(), TestHash2, buf)
 	if err == nil {
 		t.Errorf("Expected error when getting block whose put in read-only volume failed")
 	}
@@ -581,7 +668,7 @@ func testUpdateReadOnly(t TB, factory TestableVolumeFactory) {
 	}
 
 	// Overwriting an existing block in read-only volume should result in error
-	err = v.Put(TestHash, TestBlock)
+	err = v.Put(context.Background(), TestHash, TestBlock)
 	if err == nil {
 		t.Errorf("Expected error when putting block in a read-only volume")
 	}
@@ -600,7 +687,7 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) {
 	sem := make(chan int)
 	go func() {
 		buf := make([]byte, BlockSize)
-		n, err := v.Get(TestHash, buf)
+		n, err := v.Get(context.Background(), TestHash, buf)
 		if err != nil {
 			t.Errorf("err1: %v", err)
 		}
@@ -612,7 +699,7 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) {
 
 	go func() {
 		buf := make([]byte, BlockSize)
-		n, err := v.Get(TestHash2, buf)
+		n, err := v.Get(context.Background(), TestHash2, buf)
 		if err != nil {
 			t.Errorf("err2: %v", err)
 		}
@@ -624,7 +711,7 @@ func testGetConcurrent(t TB, factory TestableVolumeFactory) {
 
 	go func() {
 		buf := make([]byte, BlockSize)
-		n, err := v.Get(TestHash3, buf)
+		n, err := v.Get(context.Background(), TestHash3, buf)
 		if err != nil {
 			t.Errorf("err3: %v", err)
 		}
@@ -652,7 +739,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) {
 
 	sem := make(chan int)
 	go func(sem chan int) {
-		err := v.Put(TestHash, TestBlock)
+		err := v.Put(context.Background(), TestHash, TestBlock)
 		if err != nil {
 			t.Errorf("err1: %v", err)
 		}
@@ -660,7 +747,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) {
 	}(sem)
 
 	go func(sem chan int) {
-		err := v.Put(TestHash2, TestBlock2)
+		err := v.Put(context.Background(), TestHash2, TestBlock2)
 		if err != nil {
 			t.Errorf("err2: %v", err)
 		}
@@ -668,7 +755,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) {
 	}(sem)
 
 	go func(sem chan int) {
-		err := v.Put(TestHash3, TestBlock3)
+		err := v.Put(context.Background(), TestHash3, TestBlock3)
 		if err != nil {
 			t.Errorf("err3: %v", err)
 		}
@@ -682,7 +769,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) {
 
 	// Double check that we actually wrote the blocks we expected to write.
 	buf := make([]byte, BlockSize)
-	n, err := v.Get(TestHash, buf)
+	n, err := v.Get(context.Background(), TestHash, buf)
 	if err != nil {
 		t.Errorf("Get #1: %v", err)
 	}
@@ -690,7 +777,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) {
 		t.Errorf("Get #1: expected %s, got %s", string(TestBlock), string(buf[:n]))
 	}
 
-	n, err = v.Get(TestHash2, buf)
+	n, err = v.Get(context.Background(), TestHash2, buf)
 	if err != nil {
 		t.Errorf("Get #2: %v", err)
 	}
@@ -698,7 +785,7 @@ func testPutConcurrent(t TB, factory TestableVolumeFactory) {
 		t.Errorf("Get #2: expected %s, got %s", string(TestBlock2), string(buf[:n]))
 	}
 
-	n, err = v.Get(TestHash3, buf)
+	n, err = v.Get(context.Background(), TestHash3, buf)
 	if err != nil {
 		t.Errorf("Get #3: %v", err)
 	}
@@ -720,12 +807,12 @@ func testPutFullBlock(t TB, factory TestableVolumeFactory) {
 	wdata[0] = 'a'
 	wdata[BlockSize-1] = 'z'
 	hash := fmt.Sprintf("%x", md5.Sum(wdata))
-	err := v.Put(hash, wdata)
+	err := v.Put(context.Background(), hash, wdata)
 	if err != nil {
 		t.Fatal(err)
 	}
 	buf := make([]byte, BlockSize)
-	n, err := v.Get(hash, buf)
+	n, err := v.Get(context.Background(), hash, buf)
 	if err != nil {
 		t.Error(err)
 	}
@@ -752,7 +839,7 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) {
 	v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
 
 	buf := make([]byte, BlockSize)
-	n, err := v.Get(TestHash, buf)
+	n, err := v.Get(context.Background(), TestHash, buf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -771,7 +858,7 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) {
 			t.Fatal(err)
 		}
 	} else {
-		_, err = v.Get(TestHash, buf)
+		_, err = v.Get(context.Background(), TestHash, buf)
 		if err == nil || !os.IsNotExist(err) {
 			t.Errorf("os.IsNotExist(%v) should have been true", err)
 		}
@@ -784,7 +871,7 @@ func testTrashUntrash(t TB, factory TestableVolumeFactory) {
 	}
 
 	// Get the block - after trash and untrash sequence
-	n, err = v.Get(TestHash, buf)
+	n, err = v.Get(context.Background(), TestHash, buf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -802,7 +889,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
 
 	checkGet := func() error {
 		buf := make([]byte, BlockSize)
-		n, err := v.Get(TestHash, buf)
+		n, err := v.Get(context.Background(), TestHash, buf)
 		if err != nil {
 			return err
 		}
@@ -815,7 +902,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
 			return err
 		}
 
-		err = v.Compare(TestHash, TestBlock)
+		err = v.Compare(context.Background(), TestHash, TestBlock)
 		if err != nil {
 			return err
 		}
@@ -927,7 +1014,7 @@ func testTrashEmptyTrashUntrash(t TB, factory TestableVolumeFactory) {
 	// goes away.
 	// (In Azure volumes, un/trash changes Mtime, so first backdate again)
 	v.TouchWithDate(TestHash, time.Now().Add(-2*theConfig.BlobSignatureTTL.Duration()))
-	err = v.Trash(TestHash)
+	_ = v.Trash(TestHash)
 	err = checkGet()
 	if err == nil || !os.IsNotExist(err) {
 		t.Fatalf("os.IsNotExist(%v) should have been true", err)