9397: add normalized manifest test

[arvados.git] / sdk / go / manifest / manifest_test.go

diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index 364648d643cea64dadc706eb1f5c42e227ccbb39..0d58a9ec1116de52529b4b80ffaf1787d2ac71db 100644 (file)
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -3,6 +3,7 @@ package manifest
 import (
        "io/ioutil"
        "reflect"
+       "regexp"
        "runtime"
        "testing"
 
@@ -12,8 +13,8 @@ import (
 
 func getStackTrace() string {
        buf := make([]byte, 1000)
-       bytes_written := runtime.Stack(buf, false)
-       return "Stack Trace:\n" + string(buf[:bytes_written])
+       bytesWritten := runtime.Stack(buf, false)
+       return "Stack Trace:\n" + string(buf[:bytesWritten])
 }
 
 func expectFromChannel(t *testing.T, c <-chan string, expected string) {
@@ -60,10 +61,16 @@ func expectStringSlicesEqual(t *testing.T, actual []string, expected []string) {
        }
 }
 
+func expectFileStreamSegmentsEqual(t *testing.T, actual []FileStreamSegment, expected []FileStreamSegment) {
+       if !reflect.DeepEqual(actual, expected) {
+               t.Fatalf("Expected %v but received %v instead. %s", expected, actual, getStackTrace())
+       }
+}
+
 func expectManifestStream(t *testing.T, actual ManifestStream, expected ManifestStream) {
        expectEqual(t, actual.StreamName, expected.StreamName)
        expectStringSlicesEqual(t, actual.Blocks, expected.Blocks)
-       expectStringSlicesEqual(t, actual.FileTokens, expected.FileTokens)
+       expectFileStreamSegmentsEqual(t, actual.FileStreamSegments, expected.FileStreamSegments)
 }
 
 func expectBlockLocator(t *testing.T, actual blockdigest.BlockLocator, expected blockdigest.BlockLocator) {
@@ -75,8 +82,8 @@ func expectBlockLocator(t *testing.T, actual blockdigest.BlockLocator, expected
 func TestParseManifestStreamSimple(t *testing.T) {
        m := parseManifestStream(". 365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf 0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt")
        expectManifestStream(t, m, ManifestStream{StreamName: ".",
-               Blocks:     []string{"365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
-               FileTokens: []string{"0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}})
+               Blocks:             []string{"365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
+               FileStreamSegments: []FileStreamSegment{{0, 2310, "qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}}})
 }
 
 func TestParseBlockLocatorSimple(t *testing.T) {
@@ -84,10 +91,15 @@ func TestParseBlockLocatorSimple(t *testing.T) {
        if err != nil {
                t.Fatalf("Unexpected error parsing block locator: %v", err)
        }
-       expectBlockLocator(t, b, BlockLocator{Digest: blockdigest.AssertFromString("365f83f5f808896ec834c8b595288735"),
-               Size: 2310,
-               Hints: []string{"K@qr1hi",
-                       "Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"}})
+       d, err := blockdigest.FromString("365f83f5f808896ec834c8b595288735")
+       if err != nil {
+               t.Fatalf("Unexpected error during FromString for block locator: %v", err)
+       }
+       expectBlockLocator(t, blockdigest.BlockLocator{b.Digest, b.Size, b.Hints},
+               blockdigest.BlockLocator{Digest: d,
+                       Size: 2310,
+                       Hints: []string{"K@qr1hi",
+                               "Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"}})
 }
 
 func TestStreamIterShortManifestWithBlankStreams(t *testing.T) {
@@ -95,15 +107,15 @@ func TestStreamIterShortManifestWithBlankStreams(t *testing.T) {
        if err != nil {
                t.Fatalf("Unexpected error reading manifest from file: %v", err)
        }
-       manifest := Manifest{string(content)}
+       manifest := Manifest{Text: string(content)}
        streamIter := manifest.StreamIter()
 
        firstStream := <-streamIter
        expectManifestStream(t,
                firstStream,
                ManifestStream{StreamName: ".",
-                       Blocks:     []string{"b746e3d2104645f2f64cd3cc69dd895d+15693477+E2866e643690156651c03d876e638e674dcd79475@5441920c"},
-                       FileTokens: []string{"0:15893477:chr10_band0_s0_e3000000.fj"}})
+                       Blocks:             []string{"b746e3d2104645f2f64cd3cc69dd895d+15693477+E2866e643690156651c03d876e638e674dcd79475@5441920c"},
+                       FileStreamSegments: []FileStreamSegment{{0, 15893477, "chr10_band0_s0_e3000000.fj"}}})
 
        received, ok := <-streamIter
        if ok {
@@ -117,26 +129,33 @@ func TestBlockIterLongManifest(t *testing.T) {
        if err != nil {
                t.Fatalf("Unexpected error reading manifest from file: %v", err)
        }
-       manifest := Manifest{string(content)}
+       manifest := Manifest{Text: string(content)}
        blockChannel := manifest.BlockIterWithDuplicates()
 
        firstBlock := <-blockChannel
+       d, err := blockdigest.FromString("b746e3d2104645f2f64cd3cc69dd895d")
+       if err != nil {
+               t.Fatalf("Unexpected error during FromString for block: %v", err)
+       }
        expectBlockLocator(t,
                firstBlock,
-               blockdigest.BlockLocator{Digest: blockdigest.AssertFromString("b746e3d2104645f2f64cd3cc69dd895d"),
+               blockdigest.BlockLocator{Digest: d,
                        Size:  15693477,
                        Hints: []string{"E2866e643690156651c03d876e638e674dcd79475@5441920c"}})
        blocksRead := 1
        var lastBlock blockdigest.BlockLocator
        for lastBlock = range blockChannel {
-               //log.Printf("Blocks Read: %d", blocksRead)
                blocksRead++
        }
        expectEqual(t, blocksRead, 853)
 
+       d, err = blockdigest.FromString("f9ce82f59e5908d2d70e18df9679b469")
+       if err != nil {
+               t.Fatalf("Unexpected error during FromString for block: %v", err)
+       }
        expectBlockLocator(t,
                lastBlock,
-               blockdigest.BlockLocator{Digest: blockdigest.AssertFromString("f9ce82f59e5908d2d70e18df9679b469"),
+               blockdigest.BlockLocator{Digest: d,
                        Size:  31367794,
                        Hints: []string{"E53f903684239bcc114f7bf8ff9bd6089f33058db@5441920c"}})
 }
@@ -195,3 +214,46 @@ func TestFileSegmentIterByName(t *testing.T) {
                }
        }
 }
+
+func TestBlockIterWithBadManifest(t *testing.T) {
+       testCases := [][]string{
+               {"badstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt", "Invalid stream name: badstream"},
+               {"/badstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt", "Invalid stream name: /badstream"},
+               {". acbd18db4cc2f85cedef654fccc4a4d8+3 file1.txt", "Invalid file token: file1.txt"},
+               {". acbd18db4cc2f85cedef654fccc4a4+3 0:1:file1.txt", "No block locators found"},
+               {". acbd18db4cc2f85cedef654fccc4a4d8 0:1:file1.txt", "No block locators found"},
+               {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt file2.txt 1:2:file3.txt", "Invalid file token: file2.txt"},
+               {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt. bcde18db4cc2f85cedef654fccc4a4d8+3 1:2:file3.txt", "Invalid file token: bcde18db4cc2f85cedef654fccc4a4d8.*"},
+               {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt\n. acbd18db4cc2f85cedef654fccc4a4d8+3 ::file2.txt\n", "Invalid file token: ::file2.txt"},
+               {". acbd18db4cc2f85cedef654fccc4a4d8+3 bcde18db4cc2f85cedef654fccc4a4d8+3\n", "No file tokens found"},
+               {". acbd18db4cc2f85cedef654fccc4a4d8+3 ", "Invalid file token"},
+               {". acbd18db4cc2f85cedef654fccc4a4d8+3", "No file tokens found"},
+               {". 0:1:file1.txt\n", "No block locators found"},
+               {".\n", "No block locators found"},
+       }
+
+       for _, testCase := range testCases {
+               manifest := Manifest{Text: string(testCase[0])}
+               blockChannel := manifest.BlockIterWithDuplicates()
+
+               for block := range blockChannel {
+                       _ = block
+               }
+
+               // completed reading from blockChannel; now check for errors
+               if manifest.Err == nil {
+                       t.Fatalf("Expected error")
+               }
+
+               matched, _ := regexp.MatchString(testCase[1], manifest.Err.Error())
+               if !matched {
+                       t.Fatalf("Expected error not found. Expected: %v; Found: %v", testCase[1], manifest.Err.Error())
+               }
+       }
+}
+
+func TestNormalizeManifest(t *testing.T) {
+       m := Manifest{Text: ". acbd18db4cc2f85cedef654fccc4a4d8+40 0:10:one 10:10:one 20:10:two 30:10:two\n"}
+       normalized := m.NormalizedManifestForPath("")
+       expectEqual(t, normalized, ". acbd18db4cc2f85cedef654fccc4a4d8+40 0:20:one 20:40:two\n")
+}