sdk/go/manifest/manifest_test.go

   1 package manifest
   2
   3 import (
   4         "io/ioutil"
   5         "reflect"
   6         "runtime"
   7         "testing"
   8
   9         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  10         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
  11 )
  12
  13 func getStackTrace() string {
  14         buf := make([]byte, 1000)
  15         bytes_written := runtime.Stack(buf, false)
  16         return "Stack Trace:\n" + string(buf[:bytes_written])
  17 }
  18
  19 func expectFromChannel(t *testing.T, c <-chan string, expected string) {
  20         actual, ok := <-c
  21         if !ok {
  22                 t.Fatalf("Expected to receive %s but channel was closed. %s",
  23                         expected,
  24                         getStackTrace())
  25         }
  26         if actual != expected {
  27                 t.Fatalf("Expected %s but got %s instead. %s",
  28                         expected,
  29                         actual,
  30                         getStackTrace())
  31         }
  32 }
  33
  34 func expectChannelClosed(t *testing.T, c <-chan interface{}) {
  35         received, ok := <-c
  36         if ok {
  37                 t.Fatalf("Expected channel to be closed, but received %v instead. %s",
  38                         received,
  39                         getStackTrace())
  40         }
  41 }
  42
  43 func expectEqual(t *testing.T, actual interface{}, expected interface{}) {
  44         if actual != expected {
  45                 t.Fatalf("Expected %v but received %v instead. %s",
  46                         expected,
  47                         actual,
  48                         getStackTrace())
  49         }
  50 }
  51
  52 func expectStringSlicesEqual(t *testing.T, actual []string, expected []string) {
  53         if len(actual) != len(expected) {
  54                 t.Fatalf("Expected %v (length %d), but received %v (length %d) instead. %s", expected, len(expected), actual, len(actual), getStackTrace())
  55         }
  56         for i := range actual {
  57                 if actual[i] != expected[i] {
  58                         t.Fatalf("Expected %v but received %v instead (first disagreement at position %d). %s", expected, actual, i, getStackTrace())
  59                 }
  60         }
  61 }
  62
  63 func expectManifestStream(t *testing.T, actual ManifestStream, expected ManifestStream) {
  64         expectEqual(t, actual.StreamName, expected.StreamName)
  65         expectStringSlicesEqual(t, actual.Blocks, expected.Blocks)
  66         expectStringSlicesEqual(t, actual.FileTokens, expected.FileTokens)
  67 }
  68
  69 func expectBlockLocator(t *testing.T, actual blockdigest.BlockLocator, expected blockdigest.BlockLocator) {
  70         expectEqual(t, actual.Digest, expected.Digest)
  71         expectEqual(t, actual.Size, expected.Size)
  72         expectStringSlicesEqual(t, actual.Hints, expected.Hints)
  73 }
  74
  75 func TestParseManifestStreamSimple(t *testing.T) {
  76         m := parseManifestStream(". 365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf 0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt")
  77         expectManifestStream(t, m, ManifestStream{StreamName: ".",
  78                 Blocks:     []string{"365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
  79                 FileTokens: []string{"0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}})
  80 }
  81
  82 func TestParseBlockLocatorSimple(t *testing.T) {
  83         b, err := ParseBlockLocator("365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf")
  84         if err != nil {
  85                 t.Fatalf("Unexpected error parsing block locator: %v", err)
  86         }
  87         expectBlockLocator(t, b, BlockLocator{Digest: blockdigest.AssertFromString("365f83f5f808896ec834c8b595288735"),
  88                 Size: 2310,
  89                 Hints: []string{"K@qr1hi",
  90                         "Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"}})
  91 }
  92
  93 func TestStreamIterShortManifestWithBlankStreams(t *testing.T) {
  94         content, err := ioutil.ReadFile("testdata/short_manifest")
  95         if err != nil {
  96                 t.Fatalf("Unexpected error reading manifest from file: %v", err)
  97         }
  98         manifest := Manifest{string(content)}
  99         streamIter := manifest.StreamIter()
 100
 101         firstStream := <-streamIter
 102         expectManifestStream(t,
 103                 firstStream,
 104                 ManifestStream{StreamName: ".",
 105                         Blocks:     []string{"b746e3d2104645f2f64cd3cc69dd895d+15693477+E2866e643690156651c03d876e638e674dcd79475@5441920c"},
 106                         FileTokens: []string{"0:15893477:chr10_band0_s0_e3000000.fj"}})
 107
 108         received, ok := <-streamIter
 109         if ok {
 110                 t.Fatalf("Expected streamIter to be closed, but received %v instead.",
 111                         received)
 112         }
 113 }
 114
 115 func TestBlockIterLongManifest(t *testing.T) {
 116         content, err := ioutil.ReadFile("testdata/long_manifest")
 117         if err != nil {
 118                 t.Fatalf("Unexpected error reading manifest from file: %v", err)
 119         }
 120         manifest := Manifest{string(content)}
 121         blockChannel := manifest.BlockIterWithDuplicates()
 122
 123         firstBlock := <-blockChannel
 124         expectBlockLocator(t,
 125                 firstBlock,
 126                 blockdigest.BlockLocator{Digest: blockdigest.AssertFromString("b746e3d2104645f2f64cd3cc69dd895d"),
 127                         Size:  15693477,
 128                         Hints: []string{"E2866e643690156651c03d876e638e674dcd79475@5441920c"}})
 129         blocksRead := 1
 130         var lastBlock blockdigest.BlockLocator
 131         for lastBlock = range blockChannel {
 132                 //log.Printf("Blocks Read: %d", blocksRead)
 133                 blocksRead++
 134         }
 135         expectEqual(t, blocksRead, 853)
 136
 137         expectBlockLocator(t,
 138                 lastBlock,
 139                 blockdigest.BlockLocator{Digest: blockdigest.AssertFromString("f9ce82f59e5908d2d70e18df9679b469"),
 140                         Size:  31367794,
 141                         Hints: []string{"E53f903684239bcc114f7bf8ff9bd6089f33058db@5441920c"}})
 142 }
 143
 144 func TestUnescape(t *testing.T) {
 145         for _, testCase := range [][]string{
 146                 {`\040`, ` `},
 147                 {`\009`, `\009`},
 148                 {`\\\040\\`, `\ \`},
 149                 {`\\040\`, `\040\`},
 150         } {
 151                 in := testCase[0]
 152                 expect := testCase[1]
 153                 got := UnescapeName(in)
 154                 if expect != got {
 155                         t.Errorf("For '%s' got '%s' instead of '%s'", in, got, expect)
 156                 }
 157         }
 158 }
 159
 160 type fsegtest struct {
 161         mt   string        // manifest text
 162         f    string        // filename
 163         want []FileSegment // segments should be received on channel
 164 }
 165
 166 func TestFileSegmentIterByName(t *testing.T) {
 167         mt := arvadostest.PathologicalManifest
 168         for _, testCase := range []fsegtest{
 169                 {mt: mt, f: "zzzz", want: nil},
 170                 // This case is too sensitive: it would be acceptable
 171                 // (even preferable) to return only one empty segment.
 172                 {mt: mt, f: "foo/zero", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}, {"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 173                 {mt: mt, f: "zero@0", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 174                 {mt: mt, f: "zero@1", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 175                 {mt: mt, f: "zero@4", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 176                 {mt: mt, f: "zero@9", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 177                 {mt: mt, f: "f", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}}},
 178                 {mt: mt, f: "ooba", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}, {"37b51d194a7513e45b56f6524f2d51f2+3", 0, 2}}},
 179                 {mt: mt, f: "overlapReverse/o", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 2, 1}}},
 180                 {mt: mt, f: "overlapReverse/oo", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}}},
 181                 {mt: mt, f: "overlapReverse/ofoo", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 2, 1}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 3}}},
 182                 {mt: mt, f: "foo bar/baz", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 3}}},
 183                 // This case is too sensitive: it would be better to
 184                 // omit the empty segment.
 185                 {mt: mt, f: "segmented/frob", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}, {"37b51d194a7513e45b56f6524f2d51f2+3", 2, 1}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 1}, {"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}, {"37b51d194a7513e45b56f6524f2d51f2+3", 0, 1}}},
 186                 {mt: mt, f: "segmented/oof", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}}},
 187         } {
 188                 m := Manifest{Text: testCase.mt}
 189                 var got []FileSegment
 190                 for fs := range m.FileSegmentIterByName(testCase.f) {
 191                         got = append(got, *fs)
 192                 }
 193                 if !reflect.DeepEqual(got, testCase.want) {
 194                         t.Errorf("For %#v:\n got  %#v\n want %#v", testCase.f, got, testCase.want)
 195                 }
 196         }
 197 }