X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ebb2559b3a09636ff687316bbe512e0e8a86b168..df5c912a9eb5af7222e5446bc437ee97262542c8:/services/keepstore/azure_blob_volume_test.go

diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go
index 232382c421..85d0a1eea4 100644
--- a/services/keepstore/azure_blob_volume_test.go
+++ b/services/keepstore/azure_blob_volume_test.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
 package main
 
 import (
@@ -5,6 +9,7 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/base64"
+	"encoding/json"
 	"encoding/xml"
 	"flag"
 	"fmt"
@@ -13,6 +18,7 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"regexp"
 	"sort"
 	"strconv"
@@ -21,17 +27,24 @@ import (
 	"testing"
 	"time"
 
-	log "github.com/Sirupsen/logrus"
-	"github.com/curoverse/azure-sdk-for-go/storage"
+	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/ghodss/yaml"
+	check "gopkg.in/check.v1"
 )
 
 const (
-	// The same fake credentials used by Microsoft's Azure emulator
-	emulatorAccountName = "devstoreaccount1"
-	emulatorAccountKey  = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+	// This cannot be the fake account name "devstoreaccount1"
+	// used by Microsoft's Azure emulator: the Azure SDK
+	// recognizes that magic string and changes its behavior to
+	// cater to the Azure SDK's own test suite.
+	fakeAccountName = "fakeaccountname"
+	fakeAccountKey  = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 )
 
-var azureTestContainer string
+var (
+	azureTestContainer string
+	azureTestDebug     = os.Getenv("ARVADOS_DEBUG") != ""
+)
 
 func init() {
 	flag.StringVar(
@@ -100,7 +113,9 @@ var rangeRegexp = regexp.MustCompile(`^bytes=(\d+)-(\d+)$`)
 func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	h.Lock()
 	defer h.Unlock()
-	// defer log.Printf("azStubHandler: %+v", r)
+	if azureTestDebug {
+		defer log.Printf("azStubHandler: %+v", r)
+	}
 
 	path := strings.Split(r.URL.Path, "/")
 	container := path[1]
@@ -115,6 +130,11 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if (r.Method == "PUT" || r.Method == "POST") && r.Header.Get("Content-Length") == "" {
+		rw.WriteHeader(http.StatusLengthRequired)
+		return
+	}
+
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		return
@@ -294,7 +314,7 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 				b := storage.Blob{
 					Name: hash,
 					Properties: storage.BlobProperties{
-						LastModified:  blob.Mtime.Format(time.RFC1123),
+						LastModified:  storage.TimeRFC1123(blob.Mtime),
 						ContentLength: int64(len(blob.Data)),
 						Etag:          blob.Etag,
 					},
@@ -326,7 +346,9 @@ var localHostPortRe = regexp.MustCompile(`(127\.0\.0\.1|localhost|\[::1\]):\d+`)
 
 func (d *azStubDialer) Dial(network, address string) (net.Conn, error) {
 	if hp := localHostPortRe.FindString(address); hp != "" {
-		log.Println("azStubDialer: dial", hp, "instead of", address)
+		if azureTestDebug {
+			log.Println("azStubDialer: dial", hp, "instead of", address)
+		}
 		address = hp
 	}
 	return d.Dialer.Dial(network, address)
@@ -350,7 +372,7 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA
 		// Connect to stub instead of real Azure storage service
 		stubURLBase := strings.Split(azStub.URL, "://")[1]
 		var err error
-		if azClient, err = storage.NewClient(emulatorAccountName, emulatorAccountKey, stubURLBase, storage.DefaultAPIVersion, false); err != nil {
+		if azClient, err = storage.NewClient(fakeAccountName, fakeAccountKey, stubURLBase, storage.DefaultAPIVersion, false); err != nil {
 			t.Fatal(err)
 		}
 		container = "fakecontainername"
@@ -366,12 +388,13 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA
 		}
 	}
 
+	bs := azClient.GetBlobService()
 	v := &AzureBlobVolume{
 		ContainerName:    container,
 		ReadOnly:         readonly,
 		AzureReplication: replication,
 		azClient:         azClient,
-		bsClient:         azClient.GetBlobService(),
+		container:        &azureContainer{ctr: bs.GetContainerReference(container)},
 	}
 
 	return &TestableAzureBlobVolume{
@@ -382,6 +405,29 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA
 	}
 }
 
+var _ = check.Suite(&StubbedAzureBlobSuite{})
+
+type StubbedAzureBlobSuite struct {
+	volume            *TestableAzureBlobVolume
+	origHTTPTransport http.RoundTripper
+}
+
+func (s *StubbedAzureBlobSuite) SetUpTest(c *check.C) {
+	s.origHTTPTransport = http.DefaultTransport
+	http.DefaultTransport = &http.Transport{
+		Dial: (&azStubDialer{}).Dial,
+	}
+	azureWriteRaceInterval = time.Millisecond
+	azureWriteRacePollTime = time.Nanosecond
+
+	s.volume = NewTestableAzureBlobVolume(c, false, 3)
+}
+
+func (s *StubbedAzureBlobSuite) TearDownTest(c *check.C) {
+	s.volume.Teardown()
+	http.DefaultTransport = s.origHTTPTransport
+}
+
 func TestAzureBlobVolumeWithGeneric(t *testing.T) {
 	defer func(t http.RoundTripper) {
 		http.DefaultTransport = t
@@ -466,10 +512,10 @@ func TestAzureBlobVolumeRangeFenceposts(t *testing.T) {
 		}
 		gotHash := fmt.Sprintf("%x", md5.Sum(gotData))
 		if gotLen != size {
-			t.Error("length mismatch: got %d != %d", gotLen, size)
+			t.Errorf("length mismatch: got %d != %d", gotLen, size)
 		}
 		if gotHash != hash {
-			t.Error("hash mismatch: got %s != %s", gotHash, hash)
+			t.Errorf("hash mismatch: got %s != %s", gotHash, hash)
 		}
 	}
 }
@@ -498,9 +544,13 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
 	azureWriteRaceInterval = time.Second
 	azureWriteRacePollTime = time.Millisecond
 
-	allDone := make(chan struct{})
+	var wg sync.WaitGroup
+
 	v.azHandler.race = make(chan chan struct{})
+
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		err := v.Put(context.Background(), TestHash, TestBlock)
 		if err != nil {
 			t.Error(err)
@@ -509,21 +559,22 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) {
 	continuePut := make(chan struct{})
 	// Wait for the stub's Put to create the empty blob
 	v.azHandler.race <- continuePut
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		buf := make([]byte, len(TestBlock))
 		_, err := v.Get(context.Background(), TestHash, buf)
 		if err != nil {
 			t.Error(err)
 		}
-		close(allDone)
 	}()
 	// Wait for the stub's Get to get the empty blob
 	close(v.azHandler.race)
 	// Allow stub's Put to continue, so the real data is ready
 	// when the volume's Get retries
 	<-continuePut
-	// Wait for volume's Get to return the real data
-	<-allDone
+	// Wait for Get() and Put() to finish
+	wg.Wait()
 }
 
 func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) {
@@ -640,6 +691,48 @@ func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Contex
 	}()
 }
 
+func (s *StubbedAzureBlobSuite) TestStats(c *check.C) {
+	stats := func() string {
+		buf, err := json.Marshal(s.volume.InternalStats())
+		c.Check(err, check.IsNil)
+		return string(buf)
+	}
+
+	c.Check(stats(), check.Matches, `.*"Ops":0,.*`)
+	c.Check(stats(), check.Matches, `.*"Errors":0,.*`)
+
+	loc := "acbd18db4cc2f85cedef654fccc4a4d8"
+	_, err := s.volume.Get(context.Background(), loc, make([]byte, 3))
+	c.Check(err, check.NotNil)
+	c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`)
+	c.Check(stats(), check.Matches, `.*"Errors":[^0],.*`)
+	c.Check(stats(), check.Matches, `.*"storage\.AzureStorageServiceError 404 \(404 Not Found\)":[^0].*`)
+	c.Check(stats(), check.Matches, `.*"InBytes":0,.*`)
+
+	err = s.volume.Put(context.Background(), loc, []byte("foo"))
+	c.Check(err, check.IsNil)
+	c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`)
+	c.Check(stats(), check.Matches, `.*"CreateOps":1,.*`)
+
+	_, err = s.volume.Get(context.Background(), loc, make([]byte, 3))
+	c.Check(err, check.IsNil)
+	_, err = s.volume.Get(context.Background(), loc, make([]byte, 3))
+	c.Check(err, check.IsNil)
+	c.Check(stats(), check.Matches, `.*"InBytes":6,.*`)
+}
+
+func (s *StubbedAzureBlobSuite) TestConfig(c *check.C) {
+	var cfg Config
+	err := yaml.Unmarshal([]byte(`
+Volumes:
+  - Type: Azure
+    StorageClasses: ["class_a", "class_b"]
+`), &cfg)
+
+	c.Check(err, check.IsNil)
+	c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"})
+}
+
 func (v *TestableAzureBlobVolume) PutRaw(locator string, data []byte) {
 	v.azHandler.PutRaw(v.ContainerName, locator, data)
 }