21891: Test Extract on a large manifest.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index 090ead94bd6e02b955b7b9fd2dd3dc897299b75e..977dee37e9a067b87dedaf72106c039787cd7f36 100644 (file)
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -5,16 +5,27 @@
 package manifest
 
 import (
+       "bytes"
        "fmt"
-       "git.arvados.org/arvados.git/sdk/go/arvadostest"
-       "git.arvados.org/arvados.git/sdk/go/blockdigest"
        "io/ioutil"
+       "math/rand"
        "reflect"
        "regexp"
        "runtime"
        "testing"
+       "time"
+
+       "git.arvados.org/arvados.git/sdk/go/arvadostest"
+       "git.arvados.org/arvados.git/sdk/go/blockdigest"
+       . "gopkg.in/check.v1"
 )
 
+func TestGocheck(t *testing.T) { TestingT(t) }
+
+var _ = Suite(&suite{})
+
+type suite struct{}
+
 func getStackTrace() string {
        buf := make([]byte, 1000)
        bytesWritten := runtime.Stack(buf, false)
@@ -368,6 +379,42 @@ func TestNormalizeManifest(t *testing.T) {
        expectEqual(t, m15.Extract(".", ".").Err.Error(), "Invalid file token: 0:3bar")
 }
 
+func (s *suite) TestExtractFromLargeManifest(c *C) {
+       m := Manifest{Text: s.generateManifest(c, 200, 200, 2, 4<<20)}
+       fmt.Println(m.Extract("./dir77/file88", "./extracted").Text)
+}
+func (s *suite) generateManifest(c *C, dirCount, filesPerDir, blocksPerFile, interleaveChunk int) string {
+       c.Logf("%s building manifest with dirCount=%d filesPerDir=%d blocksPerFile=%d", time.Now(), dirCount, filesPerDir, blocksPerFile)
+       const blksize = 1 << 26
+       mb := bytes.NewBuffer(make([]byte, 0, 40000000))
+       blkid := 0
+       for i := 0; i < dirCount; i++ {
+               fmt.Fprintf(mb, "./dir%d", i)
+               for j := 0; j < filesPerDir; j++ {
+                       for k := 0; k < blocksPerFile; k++ {
+                               blkid++
+                               fmt.Fprintf(mb, " %032x+%d+A%040x@%08x", blkid, blksize, blkid, blkid)
+                       }
+               }
+               for j := 0; j < filesPerDir; j++ {
+                       if interleaveChunk == 0 {
+                               fmt.Fprintf(mb, " %d:%d:dir%d/file%d", (filesPerDir-j-1)*blocksPerFile*blksize, blocksPerFile*blksize, j, j)
+                               continue
+                       }
+                       for todo := int64(blocksPerFile) * int64(blksize); todo > 0; todo -= int64(interleaveChunk) {
+                               size := int64(interleaveChunk)
+                               if size > todo {
+                                       size = todo
+                               }
+                               offset := rand.Int63n(int64(blocksPerFile)*int64(blksize)*int64(filesPerDir) - size)
+                               fmt.Fprintf(mb, " %d:%d:dir%d/file%d", offset, size, j, j)
+                       }
+               }
+               mb.Write([]byte{'\n'})
+       }
+       return mb.String()
+}
+
 func TestFirstBlock(t *testing.T) {
        fmt.Println("ZZZ")
        expectEqual(t, firstBlock([]uint64{1, 2, 3, 4}, 3), 2)