22 "github.com/curoverse/azure-sdk-for-go/storage"
26 // The same fake credentials used by Microsoft's Azure emulator
27 emulatorAccountName = "devstoreaccount1"
28 emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
31 var azureTestContainer string
36 "test.azure-storage-container-volume",
38 "Name of Azure container to use for testing. Do not use a container with real data! Use -azure-storage-account-name and -azure-storage-key-file arguments to supply credentials.")
44 Metadata map[string]string
46 Uncommitted map[string][]byte
49 type azStubHandler struct {
51 blobs map[string]*azBlob
54 func newAzStubHandler() *azStubHandler {
55 return &azStubHandler{
56 blobs: make(map[string]*azBlob),
60 func (h *azStubHandler) TouchWithDate(container, hash string, t time.Time) {
61 if blob, ok := h.blobs[container+"|"+hash]; !ok {
68 func (h *azStubHandler) PutRaw(container, hash string, data []byte) {
71 h.blobs[container+"|"+hash] = &azBlob{
74 Uncommitted: make(map[string][]byte),
78 func (h *azStubHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
81 // defer log.Printf("azStubHandler: %+v", r)
83 path := strings.Split(r.URL.Path, "/")
90 if err := r.ParseForm(); err != nil {
91 log.Printf("azStubHandler(%+v): %s", r, err)
92 rw.WriteHeader(http.StatusBadRequest)
96 body, err := ioutil.ReadAll(r.Body)
101 type blockListRequestBody struct {
102 XMLName xml.Name `xml:"BlockList"`
106 blob, blobExists := h.blobs[container+"|"+hash]
109 case r.Method == "PUT" && r.Form.Get("comp") == "":
111 h.blobs[container+"|"+hash] = &azBlob{
114 Uncommitted: make(map[string][]byte),
117 rw.WriteHeader(http.StatusCreated)
118 case r.Method == "PUT" && r.Form.Get("comp") == "block":
121 log.Printf("Got block for nonexistent blob: %+v", r)
122 rw.WriteHeader(http.StatusBadRequest)
125 blockID, err := base64.StdEncoding.DecodeString(r.Form.Get("blockid"))
126 if err != nil || len(blockID) == 0 {
127 log.Printf("Invalid blockid: %+q", r.Form.Get("blockid"))
128 rw.WriteHeader(http.StatusBadRequest)
131 blob.Uncommitted[string(blockID)] = body
132 rw.WriteHeader(http.StatusCreated)
133 case r.Method == "PUT" && r.Form.Get("comp") == "blocklist":
134 // "Put Block List" API
135 bl := &blockListRequestBody{}
136 if err := xml.Unmarshal(body, bl); err != nil {
137 log.Printf("xml Unmarshal: %s", err)
138 rw.WriteHeader(http.StatusBadRequest)
141 for _, encBlockID := range bl.Uncommitted {
142 blockID, err := base64.StdEncoding.DecodeString(encBlockID)
143 if err != nil || len(blockID) == 0 || blob.Uncommitted[string(blockID)] == nil {
144 log.Printf("Invalid blockid: %+q", encBlockID)
145 rw.WriteHeader(http.StatusBadRequest)
148 blob.Data = blob.Uncommitted[string(blockID)]
149 blob.Etag = makeEtag()
150 blob.Mtime = time.Now()
151 delete(blob.Uncommitted, string(blockID))
153 rw.WriteHeader(http.StatusCreated)
154 case r.Method == "PUT" && r.Form.Get("comp") == "metadata":
155 // "Set Metadata Headers" API. We don't bother
156 // stubbing "Get Metadata Headers": AzureBlobVolume
157 // sets metadata headers only as a way to bump Etag
158 // and Last-Modified.
160 log.Printf("Got metadata for nonexistent blob: %+v", r)
161 rw.WriteHeader(http.StatusBadRequest)
164 blob.Metadata = make(map[string]string)
165 for k, v := range r.Header {
166 if strings.HasPrefix(strings.ToLower(k), "x-ms-meta-") {
167 blob.Metadata[k] = v[0]
170 blob.Mtime = time.Now()
171 blob.Etag = makeEtag()
172 case (r.Method == "GET" || r.Method == "HEAD") && hash != "":
175 rw.WriteHeader(http.StatusNotFound)
178 rw.Header().Set("Last-Modified", blob.Mtime.Format(time.RFC1123))
179 rw.Header().Set("Content-Length", strconv.Itoa(len(blob.Data)))
180 if r.Method == "GET" {
181 if _, err := rw.Write(blob.Data); err != nil {
182 log.Printf("write %+q: %s", blob.Data, err)
185 case r.Method == "DELETE" && hash != "":
188 rw.WriteHeader(http.StatusNotFound)
191 delete(h.blobs, container+"|"+hash)
192 rw.WriteHeader(http.StatusAccepted)
193 case r.Method == "GET" && r.Form.Get("comp") == "list" && r.Form.Get("restype") == "container":
195 prefix := container + "|" + r.Form.Get("prefix")
196 marker := r.Form.Get("marker")
199 if n, err := strconv.Atoi(r.Form.Get("maxresults")); err == nil && n >= 1 && n <= 5000 {
203 resp := storage.BlobListResponse{
206 MaxResults: int64(maxResults),
208 var hashes sort.StringSlice
209 for k := range h.blobs {
210 if strings.HasPrefix(k, prefix) {
211 hashes = append(hashes, k[len(container)+1:])
215 for _, hash := range hashes {
216 if len(resp.Blobs) == maxResults {
217 resp.NextMarker = hash
220 if len(resp.Blobs) > 0 || marker == "" || marker == hash {
221 blob := h.blobs[container+"|"+hash]
222 resp.Blobs = append(resp.Blobs, storage.Blob{
224 Properties: storage.BlobProperties{
225 LastModified: blob.Mtime.Format(time.RFC1123),
226 ContentLength: int64(len(blob.Data)),
232 buf, err := xml.Marshal(resp)
235 rw.WriteHeader(http.StatusInternalServerError)
239 log.Printf("azStubHandler: not implemented: %+v Body:%+q", r, body)
240 rw.WriteHeader(http.StatusNotImplemented)
244 // azStubDialer is a net.Dialer that notices when the Azure driver
245 // tries to connect to "devstoreaccount1.blob.127.0.0.1:46067", and
246 // in such cases transparently dials "127.0.0.1:46067" instead.
247 type azStubDialer struct {
251 var localHostPortRe = regexp.MustCompile(`(127\.0\.0\.1|localhost|\[::1\]):\d+`)
253 func (d *azStubDialer) Dial(network, address string) (net.Conn, error) {
254 if hp := localHostPortRe.FindString(address); hp != "" {
255 log.Println("azStubDialer: dial", hp, "instead of", address)
258 return d.Dialer.Dial(network, address)
261 type TestableAzureBlobVolume struct {
263 azHandler *azStubHandler
264 azStub *httptest.Server
268 func NewTestableAzureBlobVolume(t *testing.T, readonly bool, replication int) TestableVolume {
269 azHandler := newAzStubHandler()
270 azStub := httptest.NewServer(azHandler)
272 var azClient storage.Client
274 container := azureTestContainer
276 // Connect to stub instead of real Azure storage service
277 stubURLBase := strings.Split(azStub.URL, "://")[1]
279 if azClient, err = storage.NewClient(emulatorAccountName, emulatorAccountKey, stubURLBase, storage.DefaultAPIVersion, false); err != nil {
282 container = "fakecontainername"
284 // Connect to real Azure storage service
285 accountKey, err := readKeyFromFile(azureStorageAccountKeyFile)
289 azClient, err = storage.NewBasicClient(azureStorageAccountName, accountKey)
295 v := NewAzureBlobVolume(azClient, container, readonly, replication)
297 return &TestableAzureBlobVolume{
299 azHandler: azHandler,
305 func TestAzureBlobVolumeWithGeneric(t *testing.T) {
306 defer func(t http.RoundTripper) {
307 http.DefaultTransport = t
308 }(http.DefaultTransport)
309 http.DefaultTransport = &http.Transport{
310 Dial: (&azStubDialer{}).Dial,
312 DoGenericVolumeTests(t, func(t *testing.T) TestableVolume {
313 return NewTestableAzureBlobVolume(t, false, azureStorageReplication)
317 func TestReadonlyAzureBlobVolumeWithGeneric(t *testing.T) {
318 defer func(t http.RoundTripper) {
319 http.DefaultTransport = t
320 }(http.DefaultTransport)
321 http.DefaultTransport = &http.Transport{
322 Dial: (&azStubDialer{}).Dial,
324 DoGenericVolumeTests(t, func(t *testing.T) TestableVolume {
325 return NewTestableAzureBlobVolume(t, true, azureStorageReplication)
329 func TestAzureBlobVolumeReplication(t *testing.T) {
330 for r := 1; r <= 4; r++ {
331 v := NewTestableAzureBlobVolume(t, false, r)
333 if n := v.Replication(); n != r {
334 t.Errorf("Got replication %d, expected %d", n, r)
339 func (v *TestableAzureBlobVolume) PutRaw(locator string, data []byte) {
340 v.azHandler.PutRaw(v.containerName, locator, data)
343 func (v *TestableAzureBlobVolume) TouchWithDate(locator string, lastPut time.Time) {
344 v.azHandler.TouchWithDate(v.containerName, locator, lastPut)
347 func (v *TestableAzureBlobVolume) Teardown() {
351 func makeEtag() string {
352 return fmt.Sprintf("0x%x", rand.Int63())