Merge branch 'git-commit-debug' of https://github.com/wtsi-hgi/arvados

[arvados.git] / sdk / go / manifest / manifest_test.go

diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index 1fea5fb2f1b3bafc5af4fc2eed911179add47f3b..8cfe3d907721e4f7d33048dfa16ef91e38dec12d 100644 (file)
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -1,19 +1,20 @@
 package manifest
 
 import (
+       "git.curoverse.com/arvados.git/sdk/go/blockdigest"
        "io/ioutil"
        "runtime"
        "testing"
 )
 
-func getStackTrace() (string) {
+func getStackTrace() string {
        buf := make([]byte, 1000)
        bytes_written := runtime.Stack(buf, false)
        return "Stack Trace:\n" + string(buf[:bytes_written])
 }
 
 func expectFromChannel(t *testing.T, c <-chan string, expected string) {
-       actual, ok := <- c
+       actual, ok := <-c
        if !ok {
                t.Fatalf("Expected to receive %s but channel was closed. %s",
                        expected,
@@ -28,7 +29,7 @@ func expectFromChannel(t *testing.T, c <-chan string, expected string) {
 }
 
 func expectChannelClosed(t *testing.T, c <-chan interface{}) {
-       received, ok := <- c
+       received, ok := <-c
        if ok {
                t.Fatalf("Expected channel to be closed, but received %v instead. %s",
                        received,
@@ -56,73 +57,45 @@ func expectStringSlicesEqual(t *testing.T, actual []string, expected []string) {
        }
 }
 
-func expectManifestLine(t *testing.T, actual ManifestLine, expected ManifestLine) {
+func expectManifestStream(t *testing.T, actual ManifestStream, expected ManifestStream) {
        expectEqual(t, actual.StreamName, expected.StreamName)
        expectStringSlicesEqual(t, actual.Blocks, expected.Blocks)
        expectStringSlicesEqual(t, actual.Files, expected.Files)
 }
 
-func expectBlockLocator(t *testing.T, actual BlockLocator, expected BlockLocator) {
+func expectBlockLocator(t *testing.T, actual blockdigest.BlockLocator, expected blockdigest.BlockLocator) {
        expectEqual(t, actual.Digest, expected.Digest)
        expectEqual(t, actual.Size, expected.Size)
        expectStringSlicesEqual(t, actual.Hints, expected.Hints)
 }
 
-func expectLocatorPatternMatch(t *testing.T, s string) {
-       if !LocatorPattern.MatchString(s) {
-               t.Fatalf("Expected \"%s\" to match locator pattern but it did not.",
-                       s)
-       }
+func TestParseManifestStreamSimple(t *testing.T) {
+       m := parseManifestStream(". 365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf 0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt")
+       expectManifestStream(t, m, ManifestStream{StreamName: ".",
+               Blocks: []string{"365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
+               Files:  []string{"0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}})
 }
 
-func expectLocatorPatternFail(t *testing.T, s string) {
-       if LocatorPattern.MatchString(s) {
-               t.Fatalf("Expected \"%s\" to fail locator pattern but it passed.",
-                       s)
+func TestStreamIterShortManifestWithBlankStreams(t *testing.T) {
+       content, err := ioutil.ReadFile("testdata/short_manifest")
+       if err != nil {
+               t.Fatalf("Unexpected error reading manifest from file: %v", err)
        }
-}
-
-func TestLocatorPatternBasic(t *testing.T) {
-       expectLocatorPatternMatch(t, "12345678901234567890123456789012+12345")
-       expectLocatorPatternMatch(t, "A2345678901234abcdefababdeffdfdf+12345")
-       expectLocatorPatternMatch(t, "12345678901234567890123456789012+12345+A1")
-       expectLocatorPatternMatch(t,
-               "12345678901234567890123456789012+12345+A1+B123wxyz@_-")
-       expectLocatorPatternMatch(t,
-               "12345678901234567890123456789012+12345+A1+B123wxyz@_-+C@")
-
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345+")
-       expectLocatorPatternFail(t,  "1234567890123456789012345678901+12345")
-       expectLocatorPatternFail(t,  "123456789012345678901234567890123+12345")
-       expectLocatorPatternFail(t,  "g2345678901234abcdefababdeffdfdf+12345")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345 ")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345+1")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345+1A")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345+A")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345+a1")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345+A1+")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345+A1+B")
-       expectLocatorPatternFail(t,  "12345678901234567890123456789012+12345+A+B2")
-}
+       manifest := Manifest{string(content)}
+       streamIter := manifest.StreamIter()
 
-func TestParseManifestLineSimple(t *testing.T) {
-       m := parseManifestLine(". 365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf 0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt")
-       expectManifestLine(t, m, ManifestLine{StreamName: ".",
-               Blocks: []string{"365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
-               Files: []string{"0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}})
-}
+       firstStream := <-streamIter
+       expectManifestStream(t,
+               firstStream,
+               ManifestStream{StreamName: ".",
+                       Blocks: []string{"b746e3d2104645f2f64cd3cc69dd895d+15693477+E2866e643690156651c03d876e638e674dcd79475@5441920c"},
+                       Files:  []string{"0:15893477:chr10_band0_s0_e3000000.fj"}})
 
-func TestParseBlockLocatorSimple(t *testing.T) {
-       b, err := parseBlockLocator("365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf")
-       if err != nil {
-               t.Fatalf("Unexpected error parsing block locator: %v", err)
+       received, ok := <-streamIter
+       if ok {
+               t.Fatalf("Expected streamIter to be closed, but received %v instead.",
+                       received)
        }
-       expectBlockLocator(t, b, BlockLocator{Digest: "365f83f5f808896ec834c8b595288735",
-               Size: 2310,
-               Hints: []string{"K@qr1hi",
-                       "Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"}})
 }
 
 func TestBlockIterLongManifest(t *testing.T) {
@@ -136,9 +109,20 @@ func TestBlockIterLongManifest(t *testing.T) {
        firstBlock := <-blockChannel
        expectBlockLocator(t,
                firstBlock,
-               BlockLocator{Digest: "b748a3d2104645e2e84cd3cc69ddf95d",
-                       Size: 15893477,
-                       Hints: []string{"A2f66a643690158851c03df78a83fa874dcd79475@5441920c"}})
-       // TODO(misha): Add tests to check the number of blocks and that we
-       // see the last block.
+               blockdigest.BlockLocator{Digest: blockdigest.AssertFromString("b746e3d2104645f2f64cd3cc69dd895d"),
+                       Size:  15693477,
+                       Hints: []string{"E2866e643690156651c03d876e638e674dcd79475@5441920c"}})
+       blocksRead := 1
+       var lastBlock blockdigest.BlockLocator
+       for lastBlock = range blockChannel {
+               //log.Printf("Blocks Read: %d", blocksRead)
+               blocksRead++
+       }
+       expectEqual(t, blocksRead, 853)
+
+       expectBlockLocator(t,
+               lastBlock,
+               blockdigest.BlockLocator{Digest: blockdigest.AssertFromString("f9ce82f59e5908d2d70e18df9679b469"),
+                       Size:  31367794,
+                       Hints: []string{"E53f903684239bcc114f7bf8ff9bd6089f33058db@5441920c"}})
 }