sdk/go/manifest/manifest_test.go

   1 package manifest
   2
   3 import (
   4         "io/ioutil"
   5         "reflect"
   6         "regexp"
   7         "runtime"
   8         "testing"
   9
  10         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  11         "git.curoverse.com/arvados.git/sdk/go/blockdigest"
  12 )
  13
  14 func getStackTrace() string {
  15         buf := make([]byte, 1000)
  16         bytesWritten := runtime.Stack(buf, false)
  17         return "Stack Trace:\n" + string(buf[:bytesWritten])
  18 }
  19
  20 func expectFromChannel(t *testing.T, c <-chan string, expected string) {
  21         actual, ok := <-c
  22         if !ok {
  23                 t.Fatalf("Expected to receive %s but channel was closed. %s",
  24                         expected,
  25                         getStackTrace())
  26         }
  27         if actual != expected {
  28                 t.Fatalf("Expected %s but got %s instead. %s",
  29                         expected,
  30                         actual,
  31                         getStackTrace())
  32         }
  33 }
  34
  35 func expectChannelClosed(t *testing.T, c <-chan interface{}) {
  36         received, ok := <-c
  37         if ok {
  38                 t.Fatalf("Expected channel to be closed, but received %v instead. %s",
  39                         received,
  40                         getStackTrace())
  41         }
  42 }
  43
  44 func expectEqual(t *testing.T, actual interface{}, expected interface{}) {
  45         if actual != expected {
  46                 t.Fatalf("Expected %v but received %v instead. %s",
  47                         expected,
  48                         actual,
  49                         getStackTrace())
  50         }
  51 }
  52
  53 func expectStringSlicesEqual(t *testing.T, actual []string, expected []string) {
  54         if len(actual) != len(expected) {
  55                 t.Fatalf("Expected %v (length %d), but received %v (length %d) instead. %s", expected, len(expected), actual, len(actual), getStackTrace())
  56         }
  57         for i := range actual {
  58                 if actual[i] != expected[i] {
  59                         t.Fatalf("Expected %v but received %v instead (first disagreement at position %d). %s", expected, actual, i, getStackTrace())
  60                 }
  61         }
  62 }
  63
  64 func expectFileStreamSegmentsEqual(t *testing.T, actual []FileStreamSegment, expected []FileStreamSegment) {
  65         if len(actual) != len(expected) {
  66                 t.Fatalf("Expected %v (length %d), but received %v (length %d) instead. %s", expected, len(expected), actual, len(actual), getStackTrace())
  67         }
  68         for i := range actual {
  69                 if actual[i] != expected[i] {
  70                         t.Fatalf("Expected %v but received %v instead (first disagreement at position %d). %s", expected, actual, i, getStackTrace())
  71                 }
  72         }
  73 }
  74
  75 func expectManifestStream(t *testing.T, actual ManifestStream, expected ManifestStream) {
  76         expectEqual(t, actual.StreamName, expected.StreamName)
  77         expectStringSlicesEqual(t, actual.Blocks, expected.Blocks)
  78         expectFileStreamSegmentsEqual(t, actual.FileStreamSegments, expected.FileStreamSegments)
  79 }
  80
  81 func expectBlockLocator(t *testing.T, actual blockdigest.BlockLocator, expected blockdigest.BlockLocator) {
  82         expectEqual(t, actual.Digest, expected.Digest)
  83         expectEqual(t, actual.Size, expected.Size)
  84         expectStringSlicesEqual(t, actual.Hints, expected.Hints)
  85 }
  86
  87 func TestParseManifestStreamSimple(t *testing.T) {
  88         m := parseManifestStream(". 365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf 0:2310:qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt")
  89         expectManifestStream(t, m, ManifestStream{StreamName: ".",
  90                 Blocks:             []string{"365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"},
  91                 FileStreamSegments: []FileStreamSegment{{0, 2310, "qr1hi-8i9sb-ienvmpve1a0vpoi.log.txt"}}})
  92 }
  93
  94 func TestParseBlockLocatorSimple(t *testing.T) {
  95         b, err := ParseBlockLocator("365f83f5f808896ec834c8b595288735+2310+K@qr1hi+Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf")
  96         if err != nil {
  97                 t.Fatalf("Unexpected error parsing block locator: %v", err)
  98         }
  99         d, err := blockdigest.FromString("365f83f5f808896ec834c8b595288735")
 100         if err != nil {
 101                 t.Fatalf("Unexpected error during FromString for block locator: %v", err)
 102         }
 103         expectBlockLocator(t, blockdigest.BlockLocator{b.Digest, b.Size, b.Hints},
 104                 blockdigest.BlockLocator{Digest: d,
 105                         Size: 2310,
 106                         Hints: []string{"K@qr1hi",
 107                                 "Af0c9a66381f3b028677411926f0be1c6282fe67c@542b5ddf"}})
 108 }
 109
 110 func TestStreamIterShortManifestWithBlankStreams(t *testing.T) {
 111         content, err := ioutil.ReadFile("testdata/short_manifest")
 112         if err != nil {
 113                 t.Fatalf("Unexpected error reading manifest from file: %v", err)
 114         }
 115         manifest := Manifest{Text: string(content)}
 116         streamIter := manifest.StreamIter()
 117
 118         firstStream := <-streamIter
 119         expectManifestStream(t,
 120                 firstStream,
 121                 ManifestStream{StreamName: ".",
 122                         Blocks:             []string{"b746e3d2104645f2f64cd3cc69dd895d+15693477+E2866e643690156651c03d876e638e674dcd79475@5441920c"},
 123                         FileStreamSegments: []FileStreamSegment{{0, 15893477, "chr10_band0_s0_e3000000.fj"}}})
 124
 125         received, ok := <-streamIter
 126         if ok {
 127                 t.Fatalf("Expected streamIter to be closed, but received %v instead.",
 128                         received)
 129         }
 130 }
 131
 132 func TestBlockIterLongManifest(t *testing.T) {
 133         content, err := ioutil.ReadFile("testdata/long_manifest")
 134         if err != nil {
 135                 t.Fatalf("Unexpected error reading manifest from file: %v", err)
 136         }
 137         manifest := Manifest{Text: string(content)}
 138         blockChannel := manifest.BlockIterWithDuplicates()
 139
 140         firstBlock := <-blockChannel
 141         d, err := blockdigest.FromString("b746e3d2104645f2f64cd3cc69dd895d")
 142         if err != nil {
 143                 t.Fatalf("Unexpected error during FromString for block: %v", err)
 144         }
 145         expectBlockLocator(t,
 146                 firstBlock,
 147                 blockdigest.BlockLocator{Digest: d,
 148                         Size:  15693477,
 149                         Hints: []string{"E2866e643690156651c03d876e638e674dcd79475@5441920c"}})
 150         blocksRead := 1
 151         var lastBlock blockdigest.BlockLocator
 152         for lastBlock = range blockChannel {
 153                 blocksRead++
 154         }
 155         expectEqual(t, blocksRead, 853)
 156
 157         d, err = blockdigest.FromString("f9ce82f59e5908d2d70e18df9679b469")
 158         if err != nil {
 159                 t.Fatalf("Unexpected error during FromString for block: %v", err)
 160         }
 161         expectBlockLocator(t,
 162                 lastBlock,
 163                 blockdigest.BlockLocator{Digest: d,
 164                         Size:  31367794,
 165                         Hints: []string{"E53f903684239bcc114f7bf8ff9bd6089f33058db@5441920c"}})
 166 }
 167
 168 func TestUnescape(t *testing.T) {
 169         for _, testCase := range [][]string{
 170                 {`\040`, ` `},
 171                 {`\009`, `\009`},
 172                 {`\\\040\\`, `\ \`},
 173                 {`\\040\`, `\040\`},
 174         } {
 175                 in := testCase[0]
 176                 expect := testCase[1]
 177                 got := UnescapeName(in)
 178                 if expect != got {
 179                         t.Errorf("For '%s' got '%s' instead of '%s'", in, got, expect)
 180                 }
 181         }
 182 }
 183
 184 type fsegtest struct {
 185         mt   string        // manifest text
 186         f    string        // filename
 187         want []FileSegment // segments should be received on channel
 188 }
 189
 190 func TestFileSegmentIterByName(t *testing.T) {
 191         mt := arvadostest.PathologicalManifest
 192         for _, testCase := range []fsegtest{
 193                 {mt: mt, f: "zzzz", want: nil},
 194                 // This case is too sensitive: it would be acceptable
 195                 // (even preferable) to return only one empty segment.
 196                 {mt: mt, f: "foo/zero", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}, {"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 197                 {mt: mt, f: "zero@0", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 198                 {mt: mt, f: "zero@1", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 199                 {mt: mt, f: "zero@4", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 200                 {mt: mt, f: "zero@9", want: []FileSegment{{"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}}},
 201                 {mt: mt, f: "f", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}}},
 202                 {mt: mt, f: "ooba", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}, {"37b51d194a7513e45b56f6524f2d51f2+3", 0, 2}}},
 203                 {mt: mt, f: "overlapReverse/o", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 2, 1}}},
 204                 {mt: mt, f: "overlapReverse/oo", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}}},
 205                 {mt: mt, f: "overlapReverse/ofoo", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 2, 1}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 3}}},
 206                 {mt: mt, f: "foo bar/baz", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 3}}},
 207                 // This case is too sensitive: it would be better to
 208                 // omit the empty segment.
 209                 {mt: mt, f: "segmented/frob", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}, {"37b51d194a7513e45b56f6524f2d51f2+3", 2, 1}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 1}, {"d41d8cd98f00b204e9800998ecf8427e+0", 0, 0}, {"37b51d194a7513e45b56f6524f2d51f2+3", 0, 1}}},
 210                 {mt: mt, f: "segmented/oof", want: []FileSegment{{"acbd18db4cc2f85cedef654fccc4a4d8+3", 1, 2}, {"acbd18db4cc2f85cedef654fccc4a4d8+3", 0, 1}}},
 211         } {
 212                 m := Manifest{Text: testCase.mt}
 213                 var got []FileSegment
 214                 for fs := range m.FileSegmentIterByName(testCase.f) {
 215                         got = append(got, *fs)
 216                 }
 217                 if !reflect.DeepEqual(got, testCase.want) {
 218                         t.Errorf("For %#v:\n got  %#v\n want %#v", testCase.f, got, testCase.want)
 219                 }
 220         }
 221 }
 222
 223 func TestBlockIterWithBadManifest(t *testing.T) {
 224         testCases := [][]string{
 225                 {"badstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt", "Invalid stream name: badstream"},
 226                 {"/badstream acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt", "Invalid stream name: /badstream"},
 227                 {". acbd18db4cc2f85cedef654fccc4a4d8+3 file1.txt", "Invalid file token: file1.txt"},
 228                 {". acbd18db4cc2f85cedef654fccc4a4+3 0:1:file1.txt", "No block locators found"},
 229                 {". acbd18db4cc2f85cedef654fccc4a4d8 0:1:file1.txt", "No block locators found"},
 230                 {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt file2.txt 1:2:file3.txt", "Invalid file token: file2.txt"},
 231                 {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt. bcde18db4cc2f85cedef654fccc4a4d8+3 1:2:file3.txt", "Invalid file token: bcde18db4cc2f85cedef654fccc4a4d8.*"},
 232                 {". acbd18db4cc2f85cedef654fccc4a4d8+3 0:1:file1.txt\n. acbd18db4cc2f85cedef654fccc4a4d8+3 ::file2.txt\n", "Invalid file token: ::file2.txt"},
 233                 {". acbd18db4cc2f85cedef654fccc4a4d8+3 bcde18db4cc2f85cedef654fccc4a4d8+3\n", "No file tokens found"},
 234                 {". acbd18db4cc2f85cedef654fccc4a4d8+3 ", "Invalid file token"},
 235                 {". acbd18db4cc2f85cedef654fccc4a4d8+3", "No file tokens found"},
 236                 {". 0:1:file1.txt\n", "No block locators found"},
 237                 {".\n", "No block locators found"},
 238         }
 239
 240         for _, testCase := range testCases {
 241                 manifest := Manifest{Text: string(testCase[0])}
 242                 blockChannel := manifest.BlockIterWithDuplicates()
 243
 244                 for block := range blockChannel {
 245                         _ = block
 246                 }
 247
 248                 // completed reading from blockChannel; now check for errors
 249                 if manifest.Err == nil {
 250                         t.Fatalf("Expected error")
 251                 }
 252
 253                 matched, _ := regexp.MatchString(testCase[1], manifest.Err.Error())
 254                 if !matched {
 255                         t.Fatalf("Expected error not found. Expected: %v; Found: %v", testCase[1], manifest.Err.Error())
 256                 }
 257         }
 258 }