X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c7259164cd07f09b5d63f39ddc850c50f249ded2..472d5d0daa87835ab04c02a6298fdc338d1a0446:/services/keepstore/azure_blob_volume_test.go diff --git a/services/keepstore/azure_blob_volume_test.go b/services/keepstore/azure_blob_volume_test.go index e3c0e27083..85d0a1eea4 100644 --- a/services/keepstore/azure_blob_volume_test.go +++ b/services/keepstore/azure_blob_volume_test.go @@ -1,18 +1,24 @@ +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + package main import ( "bytes" + "context" "crypto/md5" "encoding/base64" + "encoding/json" "encoding/xml" "flag" "fmt" "io/ioutil" - "log" "math/rand" "net" "net/http" "net/http/httptest" + "os" "regexp" "sort" "strconv" @@ -21,16 +27,24 @@ import ( "testing" "time" - "github.com/curoverse/azure-sdk-for-go/storage" + "github.com/Azure/azure-sdk-for-go/storage" + "github.com/ghodss/yaml" + check "gopkg.in/check.v1" ) const ( - // The same fake credentials used by Microsoft's Azure emulator - emulatorAccountName = "devstoreaccount1" - emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + // This cannot be the fake account name "devstoreaccount1" + // used by Microsoft's Azure emulator: the Azure SDK + // recognizes that magic string and changes its behavior to + // cater to the Azure SDK's own test suite. + fakeAccountName = "fakeaccountname" + fakeAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" ) -var azureTestContainer string +var ( + azureTestContainer string + azureTestDebug = os.Getenv("ARVADOS_DEBUG") != "" +) func init() { flag.StringVar( @@ -74,6 +88,7 @@ func (h *azStubHandler) PutRaw(container, hash string, data []byte) { h.blobs[container+"|"+hash] = &azBlob{ Data: data, Mtime: time.Now(), + Metadata: make(map[string]string), Uncommitted: make(map[string][]byte), } } @@ -98,7 +113,9 @@ var rangeRegexp = regexp.MustCompile(`^bytes=(\d+)-(\d+)$`) func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { h.Lock() defer h.Unlock() - // defer log.Printf("azStubHandler: %+v", r) + if azureTestDebug { + defer log.Printf("azStubHandler: %+v", r) + } path := strings.Split(r.URL.Path, "/") container := path[1] @@ -113,6 +130,11 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { return } + if (r.Method == "PUT" || r.Method == "POST") && r.Header.Get("Content-Length") == "" { + rw.WriteHeader(http.StatusLengthRequired) + return + } + body, err := ioutil.ReadAll(r.Body) if err != nil { return @@ -136,14 +158,23 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { h.blobs[container+"|"+hash] = &azBlob{ Mtime: time.Now(), Uncommitted: make(map[string][]byte), + Metadata: make(map[string]string), Etag: makeEtag(), } h.unlockAndRace() } + metadata := make(map[string]string) + for k, v := range r.Header { + if strings.HasPrefix(strings.ToLower(k), "x-ms-meta-") { + name := k[len("x-ms-meta-"):] + metadata[strings.ToLower(name)] = v[0] + } + } h.blobs[container+"|"+hash] = &azBlob{ Data: body, Mtime: time.Now(), Uncommitted: make(map[string][]byte), + Metadata: metadata, Etag: makeEtag(), } rw.WriteHeader(http.StatusCreated) @@ -196,11 +227,22 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { blob.Metadata = make(map[string]string) for k, v := range r.Header { if strings.HasPrefix(strings.ToLower(k), "x-ms-meta-") { - blob.Metadata[k] = v[0] + name := k[len("x-ms-meta-"):] + blob.Metadata[strings.ToLower(name)] = v[0] } } blob.Mtime = time.Now() blob.Etag = makeEtag() + case (r.Method == "GET" || r.Method == "HEAD") && r.Form.Get("comp") == "metadata" && hash != "": + // "Get Blob Metadata" API + if !blobExists { + rw.WriteHeader(http.StatusNotFound) + return + } + for k, v := range blob.Metadata { + rw.Header().Set(fmt.Sprintf("x-ms-meta-%s", k), v) + } + return case (r.Method == "GET" || r.Method == "HEAD") && hash != "": // "Get Blob" API if !blobExists { @@ -265,14 +307,20 @@ func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { } if len(resp.Blobs) > 0 || marker == "" || marker == hash { blob := h.blobs[container+"|"+hash] - resp.Blobs = append(resp.Blobs, storage.Blob{ + bmeta := map[string]string(nil) + if r.Form.Get("include") == "metadata" { + bmeta = blob.Metadata + } + b := storage.Blob{ Name: hash, Properties: storage.BlobProperties{ - LastModified: blob.Mtime.Format(time.RFC1123), + LastModified: storage.TimeRFC1123(blob.Mtime), ContentLength: int64(len(blob.Data)), Etag: blob.Etag, }, - }) + Metadata: bmeta, + } + resp.Blobs = append(resp.Blobs, b) } } buf, err := xml.Marshal(resp) @@ -298,7 +346,9 @@ var localHostPortRe = regexp.MustCompile(`(127\.0\.0\.1|localhost|\[::1\]):\d+`) func (d *azStubDialer) Dial(network, address string) (net.Conn, error) { if hp := localHostPortRe.FindString(address); hp != "" { - log.Println("azStubDialer: dial", hp, "instead of", address) + if azureTestDebug { + log.Println("azStubDialer: dial", hp, "instead of", address) + } address = hp } return d.Dialer.Dial(network, address) @@ -322,7 +372,7 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA // Connect to stub instead of real Azure storage service stubURLBase := strings.Split(azStub.URL, "://")[1] var err error - if azClient, err = storage.NewClient(emulatorAccountName, emulatorAccountKey, stubURLBase, storage.DefaultAPIVersion, false); err != nil { + if azClient, err = storage.NewClient(fakeAccountName, fakeAccountKey, stubURLBase, storage.DefaultAPIVersion, false); err != nil { t.Fatal(err) } container = "fakecontainername" @@ -338,7 +388,14 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA } } - v := NewAzureBlobVolume(azClient, container, readonly, replication) + bs := azClient.GetBlobService() + v := &AzureBlobVolume{ + ContainerName: container, + ReadOnly: readonly, + AzureReplication: replication, + azClient: azClient, + container: &azureContainer{ctr: bs.GetContainerReference(container)}, + } return &TestableAzureBlobVolume{ AzureBlobVolume: v, @@ -348,6 +405,29 @@ func NewTestableAzureBlobVolume(t TB, readonly bool, replication int) *TestableA } } +var _ = check.Suite(&StubbedAzureBlobSuite{}) + +type StubbedAzureBlobSuite struct { + volume *TestableAzureBlobVolume + origHTTPTransport http.RoundTripper +} + +func (s *StubbedAzureBlobSuite) SetUpTest(c *check.C) { + s.origHTTPTransport = http.DefaultTransport + http.DefaultTransport = &http.Transport{ + Dial: (&azStubDialer{}).Dial, + } + azureWriteRaceInterval = time.Millisecond + azureWriteRacePollTime = time.Nanosecond + + s.volume = NewTestableAzureBlobVolume(c, false, 3) +} + +func (s *StubbedAzureBlobSuite) TearDownTest(c *check.C) { + s.volume.Teardown() + http.DefaultTransport = s.origHTTPTransport +} + func TestAzureBlobVolumeWithGeneric(t *testing.T) { defer func(t http.RoundTripper) { http.DefaultTransport = t @@ -421,21 +501,21 @@ func TestAzureBlobVolumeRangeFenceposts(t *testing.T) { data[i] = byte((i + 7) & 0xff) } hash := fmt.Sprintf("%x", md5.Sum(data)) - err := v.Put(hash, data) + err := v.Put(context.Background(), hash, data) if err != nil { t.Error(err) } gotData := make([]byte, len(data)) - gotLen, err := v.Get(hash, gotData) + gotLen, err := v.Get(context.Background(), hash, gotData) if err != nil { t.Error(err) } gotHash := fmt.Sprintf("%x", md5.Sum(gotData)) if gotLen != size { - t.Error("length mismatch: got %d != %d", gotLen, size) + t.Errorf("length mismatch: got %d != %d", gotLen, size) } if gotHash != hash { - t.Error("hash mismatch: got %s != %s", gotHash, hash) + t.Errorf("hash mismatch: got %s != %s", gotHash, hash) } } } @@ -464,10 +544,14 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) { azureWriteRaceInterval = time.Second azureWriteRacePollTime = time.Millisecond - allDone := make(chan struct{}) + var wg sync.WaitGroup + v.azHandler.race = make(chan chan struct{}) + + wg.Add(1) go func() { - err := v.Put(TestHash, TestBlock) + defer wg.Done() + err := v.Put(context.Background(), TestHash, TestBlock) if err != nil { t.Error(err) } @@ -475,21 +559,22 @@ func TestAzureBlobVolumeCreateBlobRace(t *testing.T) { continuePut := make(chan struct{}) // Wait for the stub's Put to create the empty blob v.azHandler.race <- continuePut + wg.Add(1) go func() { + defer wg.Done() buf := make([]byte, len(TestBlock)) - _, err := v.Get(TestHash, buf) + _, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Error(err) } - close(allDone) }() // Wait for the stub's Get to get the empty blob close(v.azHandler.race) // Allow stub's Put to continue, so the real data is ready // when the volume's Get retries <-continuePut - // Wait for volume's Get to return the real data - <-allDone + // Wait for Get() and Put() to finish + wg.Wait() } func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) { @@ -520,7 +605,7 @@ func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) { go func() { defer close(allDone) buf := make([]byte, BlockSize) - n, err := v.Get(TestHash, buf) + n, err := v.Get(context.Background(), TestHash, buf) if err != nil { t.Error(err) return @@ -542,12 +627,118 @@ func TestAzureBlobVolumeCreateBlobRaceDeadline(t *testing.T) { } } +func TestAzureBlobVolumeContextCancelGet(t *testing.T) { + testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error { + v.PutRaw(TestHash, TestBlock) + _, err := v.Get(ctx, TestHash, make([]byte, BlockSize)) + return err + }) +} + +func TestAzureBlobVolumeContextCancelPut(t *testing.T) { + testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error { + return v.Put(ctx, TestHash, make([]byte, BlockSize)) + }) +} + +func TestAzureBlobVolumeContextCancelCompare(t *testing.T) { + testAzureBlobVolumeContextCancel(t, func(ctx context.Context, v *TestableAzureBlobVolume) error { + v.PutRaw(TestHash, TestBlock) + return v.Compare(ctx, TestHash, TestBlock2) + }) +} + +func testAzureBlobVolumeContextCancel(t *testing.T, testFunc func(context.Context, *TestableAzureBlobVolume) error) { + defer func(t http.RoundTripper) { + http.DefaultTransport = t + }(http.DefaultTransport) + http.DefaultTransport = &http.Transport{ + Dial: (&azStubDialer{}).Dial, + } + + v := NewTestableAzureBlobVolume(t, false, 3) + defer v.Teardown() + v.azHandler.race = make(chan chan struct{}) + + ctx, cancel := context.WithCancel(context.Background()) + allDone := make(chan struct{}) + go func() { + defer close(allDone) + err := testFunc(ctx, v) + if err != context.Canceled { + t.Errorf("got %T %q, expected %q", err, err, context.Canceled) + } + }() + releaseHandler := make(chan struct{}) + select { + case <-allDone: + t.Error("testFunc finished without waiting for v.azHandler.race") + case <-time.After(10 * time.Second): + t.Error("timed out waiting to enter handler") + case v.azHandler.race <- releaseHandler: + } + + cancel() + + select { + case <-time.After(10 * time.Second): + t.Error("timed out waiting to cancel") + case <-allDone: + } + + go func() { + <-releaseHandler + }() +} + +func (s *StubbedAzureBlobSuite) TestStats(c *check.C) { + stats := func() string { + buf, err := json.Marshal(s.volume.InternalStats()) + c.Check(err, check.IsNil) + return string(buf) + } + + c.Check(stats(), check.Matches, `.*"Ops":0,.*`) + c.Check(stats(), check.Matches, `.*"Errors":0,.*`) + + loc := "acbd18db4cc2f85cedef654fccc4a4d8" + _, err := s.volume.Get(context.Background(), loc, make([]byte, 3)) + c.Check(err, check.NotNil) + c.Check(stats(), check.Matches, `.*"Ops":[^0],.*`) + c.Check(stats(), check.Matches, `.*"Errors":[^0],.*`) + c.Check(stats(), check.Matches, `.*"storage\.AzureStorageServiceError 404 \(404 Not Found\)":[^0].*`) + c.Check(stats(), check.Matches, `.*"InBytes":0,.*`) + + err = s.volume.Put(context.Background(), loc, []byte("foo")) + c.Check(err, check.IsNil) + c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`) + c.Check(stats(), check.Matches, `.*"CreateOps":1,.*`) + + _, err = s.volume.Get(context.Background(), loc, make([]byte, 3)) + c.Check(err, check.IsNil) + _, err = s.volume.Get(context.Background(), loc, make([]byte, 3)) + c.Check(err, check.IsNil) + c.Check(stats(), check.Matches, `.*"InBytes":6,.*`) +} + +func (s *StubbedAzureBlobSuite) TestConfig(c *check.C) { + var cfg Config + err := yaml.Unmarshal([]byte(` +Volumes: + - Type: Azure + StorageClasses: ["class_a", "class_b"] +`), &cfg) + + c.Check(err, check.IsNil) + c.Check(cfg.Volumes[0].GetStorageClasses(), check.DeepEquals, []string{"class_a", "class_b"}) +} + func (v *TestableAzureBlobVolume) PutRaw(locator string, data []byte) { - v.azHandler.PutRaw(v.containerName, locator, data) + v.azHandler.PutRaw(v.ContainerName, locator, data) } func (v *TestableAzureBlobVolume) TouchWithDate(locator string, lastPut time.Time) { - v.azHandler.TouchWithDate(v.containerName, locator, lastPut) + v.azHandler.TouchWithDate(v.ContainerName, locator, lastPut) } func (v *TestableAzureBlobVolume) Teardown() {