From: Tom Clegg <tom@tomclegg.ca>
Date: Tue, 15 Jun 2021 13:03:23 +0000 (-0400)
Subject: Fix "gob: encoder: message too big"
X-Git-Url: https://git.arvados.org/lightning.git/commitdiff_plain/ca9554c61f123ae1db10d89cdd17588d0ee2a47f

Fix "gob: encoder: message too big"

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
---

diff --git a/flake.go b/flake.go
index d880350a4a..20f25c78ce 100644
--- a/flake.go
+++ b/flake.go
@@ -51,7 +51,7 @@ func (cmd *flakecmd) RunCommand(prog string, args []string, stdin io.Reader, std
 			Name:        "lightning flake",
 			Client:      arvados.NewClientFromEnv(),
 			ProjectUUID: *projectUUID,
-			RAM:         500000000000,
+			RAM:         700000000000,
 			VCPUs:       96,
 			Priority:    *priority,
 			KeepCache:   2,
diff --git a/tilelib.go b/tilelib.go
index ace24cec0a..baeb259ea0 100644
--- a/tilelib.go
+++ b/tilelib.go
@@ -323,30 +323,35 @@ func (tilelib *tileLibrary) WriteDir(dir string) error {
 	for start := range files {
 		start := start
 		go func() {
-			ent0 := LibraryEntry{
-				TagSet: tilelib.taglib.Tags(),
+			err := encoders[start].Encode(LibraryEntry{TagSet: tilelib.taglib.Tags()})
+			if err != nil {
+				errs <- err
+				return
 			}
 			if start == 0 {
 				// For now, just write all the genomes and refs
 				// to the first file
 				for name, cg := range tilelib.compactGenomes {
-					ent0.CompactGenomes = append(ent0.CompactGenomes, CompactGenome{
+					err := encoders[start].Encode(LibraryEntry{CompactGenomes: []CompactGenome{{
 						Name:     name,
 						Variants: cg,
-					})
+					}}})
+					if err != nil {
+						errs <- err
+						return
+					}
 				}
 				for name, tseqs := range tilelib.refseqs {
-					ent0.CompactSequences = append(ent0.CompactSequences, CompactSequence{
+					err := encoders[start].Encode(LibraryEntry{CompactSequences: []CompactSequence{{
 						Name:          name,
 						TileSequences: tseqs,
-					})
+					}}})
+					if err != nil {
+						errs <- err
+						return
+					}
 				}
 			}
-			err := encoders[start].Encode(ent0)
-			if err != nil {
-				errs <- err
-				return
-			}
 			tvs := []TileVariant{}
 			for tag := start; tag < len(tilelib.variant) && ctx.Err() == nil; tag += nfiles {
 				tvs = tvs[:0]