12483: Compress adjacent segments when writing filetokens.

"0:100:foo 100:100:foo 200:100:foo" --> "0:300:foo"

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/sdk/go/arvados/collection_fs.go b/sdk/go/arvados/collection_fs.go
index 3cae9cbbb5d7bc2d28f5f4d647b59ba61867878b..beddb9249b379ca0ec110b95bbd2ace0e9c94aff 100644 (file)
--- a/sdk/go/arvados/collection_fs.go
+++ b/sdk/go/arvados/collection_fs.go
@@ -725,11 +725,18 @@ func (dn *dirnode) marshalManifest(prefix string) (string, error) {
                                        } else {
                                                blocks = append(blocks, e.locator)
                                        }
-                                       segments = append(segments, m1segment{
+                                       next := m1segment{
                                                name:   name,
                                                offset: streamLen + int64(e.offset),
                                                length: int64(e.length),
-                                       })
+                                       }
+                                       if prev := len(segments) - 1; prev >= 0 &&
+                                               segments[prev].name == name &&
+                                               segments[prev].offset+segments[prev].length == next.offset {
+                                               segments[prev].length += next.length
+                                       } else {
+                                               segments = append(segments, next)
+                                       }
                                        streamLen += int64(e.size)
                                default:
                                        // This can't happen: we

diff --git a/sdk/go/arvados/collection_fs_test.go b/sdk/go/arvados/collection_fs_test.go
index 324ece11ceff3ddd9ccdc578fe518bf5a229cf80..6e8e6152542f7c4c959e45ec384e8263c88454a5 100644 (file)
--- a/sdk/go/arvados/collection_fs_test.go
+++ b/sdk/go/arvados/collection_fs_test.go
@@ -738,7 +738,7 @@ func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) {
        c.Check(currentMemExtents(), check.HasLen, 1)
 
        m, err := fs.MarshalManifest(".")
-       c.Check(m, check.Not(check.Equals), "")
+       c.Check(m, check.Matches, `[^:]* 0:50000:50K\n`)
        c.Check(err, check.IsNil)
        c.Check(currentMemExtents(), check.HasLen, 0)
 }