Fix some entries skipped by WriteDir.

[lightning.git] / tilelib.go

diff --git a/tilelib.go b/tilelib.go
index cee592ba7df1029064c194b5153e4660ff2d1183..ea1cb7ea780b17038ad185bb14ba2b8aefa1e56f 100644 (file)
--- a/tilelib.go
+++ b/tilelib.go
@@ -174,8 +174,9 @@ func (tilelib *tileLibrary) loadCompactGenomes(cgs []CompactGenome, variantmap m
 }
 
 func (tilelib *tileLibrary) loadCompactSequences(cseqs []CompactSequence, variantmap map[tileLibRef]tileVariantID) error {
-       log.Debugf("loadCompactSequences: %d", len(cseqs))
+       log.Infof("loadCompactSequences: %d todo", len(cseqs))
        for _, cseq := range cseqs {
+               log.Infof("loadCompactSequences: checking %s", cseq.Name)
                for _, tseq := range cseq.TileSequences {
                        for i, libref := range tseq {
                                if libref.Variant == 0 {
@@ -198,6 +199,7 @@ func (tilelib *tileLibrary) loadCompactSequences(cseqs []CompactSequence, varian
                                return err
                        }
                }
+               log.Infof("loadCompactSequences: checking %s done", cseq.Name)
        }
        tilelib.mtx.Lock()
        defer tilelib.mtx.Unlock()
@@ -207,6 +209,7 @@ func (tilelib *tileLibrary) loadCompactSequences(cseqs []CompactSequence, varian
        for _, cseq := range cseqs {
                tilelib.refseqs[cseq.Name] = cseq.TileSequences
        }
+       log.Info("loadCompactSequences: done")
        return nil
 }
 
@@ -265,7 +268,7 @@ func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGeno
                        var variantmap = map[tileLibRef]tileVariantID{}
                        var cgs []CompactGenome
                        var cseqs []CompactSequence
-                       errs <- DecodeLibrary(f, strings.HasSuffix(path, ".gz"), func(ent *LibraryEntry) error {
+                       err = DecodeLibrary(f, strings.HasSuffix(path, ".gz"), func(ent *LibraryEntry) error {
                                if ctx.Err() != nil {
                                        return ctx.Err()
                                }
@@ -296,6 +299,7 @@ func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGeno
                        for k, v := range variantmap {
                                allvariantmap[k] = v
                        }
+                       errs <- err
                }()
        }
        for range files {
@@ -330,7 +334,8 @@ func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGeno
 }
 
 func (tilelib *tileLibrary) WriteDir(dir string) error {
-       nfiles := 128
+       ntilefiles := 128
+       nfiles := ntilefiles + len(tilelib.refseqs)
        files := make([]*os.File, nfiles)
        for i := range files {
                f, err := os.OpenFile(fmt.Sprintf("%s/library.%04d.gob.gz", dir, i), os.O_CREATE|os.O_WRONLY, 0666)
@@ -360,6 +365,12 @@ func (tilelib *tileLibrary) WriteDir(dir string) error {
        }
        sort.Strings(cgnames)
 
+       refnames := make([]string, 0, len(tilelib.refseqs))
+       for name := range tilelib.refseqs {
+               refnames = append(refnames, name)
+       }
+       sort.Strings(refnames)
+
        log.Infof("WriteDir: writing %d files", nfiles)
        ctx, cancel := context.WithCancel(context.Background())
        defer cancel()
@@ -372,21 +383,17 @@ func (tilelib *tileLibrary) WriteDir(dir string) error {
                                errs <- err
                                return
                        }
-                       if start == 0 {
-                               // For now, just write all the refs to
-                               // the first file
-                               for name, tseqs := range tilelib.refseqs {
-                                       err := encoders[start].Encode(LibraryEntry{CompactSequences: []CompactSequence{{
-                                               Name:          name,
-                                               TileSequences: tseqs,
-                                       }}})
-                                       if err != nil {
-                                               errs <- err
-                                               return
-                                       }
-                               }
+                       if refidx := start - ntilefiles; refidx >= 0 {
+                               // write each ref to its own file
+                               // (they seem to load very slowly)
+                               name := refnames[refidx]
+                               errs <- encoders[start].Encode(LibraryEntry{CompactSequences: []CompactSequence{{
+                                       Name:          name,
+                                       TileSequences: tilelib.refseqs[name],
+                               }}})
+                               return
                        }
-                       for i := start; i < len(cgnames); i += nfiles {
+                       for i := start; i < len(cgnames); i += ntilefiles {
                                err := encoders[start].Encode(LibraryEntry{CompactGenomes: []CompactGenome{{
                                        Name:     cgnames[i],
                                        Variants: tilelib.compactGenomes[cgnames[i]],
@@ -397,7 +404,7 @@ func (tilelib *tileLibrary) WriteDir(dir string) error {
                                }
                        }
                        tvs := []TileVariant{}
-                       for tag := start; tag < len(tilelib.variant) && ctx.Err() == nil; tag += nfiles {
+                       for tag := start; tag < len(tilelib.variant) && ctx.Err() == nil; tag += ntilefiles {
                                tvs = tvs[:0]
                                for idx, hash := range tilelib.variant[tag] {
                                        tvs = append(tvs, TileVariant{