1 // Copyright (C) The Lightning Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
22 "github.com/klauspost/pgzip"
23 log "github.com/sirupsen/logrus"
24 "golang.org/x/crypto/blake2b"
27 type tileVariantID uint16 // 1-based
29 type tileLibRef struct {
34 type tileSeq map[string][]tileLibRef
36 func (tseq tileSeq) Variants() ([]tileVariantID, int, int) {
38 for _, refs := range tseq {
39 for _, ref := range refs {
40 if maxtag < int(ref.Tag) {
45 vars := make([]tileVariantID, maxtag+1)
47 for _, refs := range tseq {
48 for _, ref := range refs {
49 if vars[int(ref.Tag)] != 0 {
54 vars[int(ref.Tag)] = ref.Variant
57 return vars, kept, dropped
60 type tileLibrary struct {
63 retainTileSequences bool
66 variant [][][blake2b.Size256]byte
67 refseqs map[string]map[string][]tileLibRef
68 compactGenomes map[string][]tileVariantID
69 seq2 map[[2]byte]map[[blake2b.Size256]byte][]byte
70 seq2lock map[[2]byte]sync.Locker
72 // if non-nil, write out any tile variants added while tiling
75 onAddTileVariant func(libref tileLibRef, hash [blake2b.Size256]byte, seq []byte) error
76 onAddGenome func(CompactGenome) error
77 onAddRefseq func(CompactSequence) error
83 func (tilelib *tileLibrary) loadTagSet(newtagset [][]byte) error {
84 // Loading a tagset means either passing it through to the
85 // output (if it's the first one we've seen), or just ensuring
86 // it doesn't disagree with what we already have.
87 if len(newtagset) == 0 {
91 defer tilelib.mtx.Unlock()
92 if tilelib.taglib == nil || tilelib.taglib.Len() == 0 {
93 tilelib.taglib = &tagLibrary{}
94 err := tilelib.taglib.setTags(newtagset)
98 if tilelib.encoder != nil {
99 err = tilelib.encoder.Encode(LibraryEntry{
106 } else if tilelib.taglib.Len() != len(newtagset) {
107 return fmt.Errorf("cannot merge libraries with differing tagsets")
109 current := tilelib.taglib.Tags()
110 for i := range newtagset {
111 if !bytes.Equal(newtagset[i], current[i]) {
112 return fmt.Errorf("cannot merge libraries with differing tagsets")
119 func (tilelib *tileLibrary) loadTileVariants(tvs []TileVariant, variantmap map[tileLibRef]tileVariantID) error {
120 for _, tv := range tvs {
121 // Assign a new variant ID (unique across all inputs)
122 // for each input variant.
123 variantmap[tileLibRef{Tag: tv.Tag, Variant: tv.Variant}] = tilelib.getRef(tv.Tag, tv.Sequence).Variant
128 func (tilelib *tileLibrary) loadCompactGenomes(cgs []CompactGenome, variantmap map[tileLibRef]tileVariantID) error {
129 log.Debugf("loadCompactGenomes: %d", len(cgs))
130 var wg sync.WaitGroup
131 errs := make(chan error, 1)
132 for _, cg := range cgs {
137 for i, variant := range cg.Variants {
145 newvariant, ok := variantmap[tileLibRef{Tag: tag, Variant: variant}]
147 err := fmt.Errorf("oops: genome %q has variant %d for tag %d, but that variant was not in its library", cg.Name, variant, tag)
154 // log.Tracef("loadCompactGenomes: cg %s tag %d variant %d => %d", cg.Name, tag, variant, newvariant)
155 cg.Variants[i] = newvariant
157 if tilelib.onAddGenome != nil {
158 err := tilelib.onAddGenome(cg)
167 if tilelib.encoder != nil {
168 err := tilelib.encoder.Encode(LibraryEntry{
169 CompactGenomes: []CompactGenome{cg},
179 if tilelib.compactGenomes != nil {
181 defer tilelib.mtx.Unlock()
182 tilelib.compactGenomes[cg.Name] = cg.Variants
191 func (tilelib *tileLibrary) loadCompactSequences(cseqs []CompactSequence, variantmap map[tileLibRef]tileVariantID) error {
192 log.Infof("loadCompactSequences: %d todo", len(cseqs))
193 for _, cseq := range cseqs {
194 log.Infof("loadCompactSequences: checking %s", cseq.Name)
195 for _, tseq := range cseq.TileSequences {
196 for i, libref := range tseq {
197 if libref.Variant == 0 {
198 // No variant (e.g., import
199 // dropped tiles with
200 // no-calls) = no translation.
203 v, ok := variantmap[libref]
205 return fmt.Errorf("oops: CompactSequence %q has variant %d for tag %d, but that variant was not in its library", cseq.Name, libref.Variant, libref.Tag)
210 if tilelib.encoder != nil {
211 if err := tilelib.encoder.Encode(LibraryEntry{
212 CompactSequences: []CompactSequence{cseq},
217 if tilelib.onAddRefseq != nil {
218 err := tilelib.onAddRefseq(cseq)
223 log.Infof("loadCompactSequences: checking %s done", cseq.Name)
226 defer tilelib.mtx.Unlock()
227 if tilelib.refseqs == nil {
228 tilelib.refseqs = map[string]map[string][]tileLibRef{}
230 for _, cseq := range cseqs {
231 tilelib.refseqs[cseq.Name] = cseq.TileSequences
233 log.Info("loadCompactSequences: done")
237 func allGobFiles(path string) ([]string, error) {
244 fis, err := f.Readdir(-1)
246 return []string{path}, nil
248 for _, fi := range fis {
249 if fi.Name() == "." || fi.Name() == ".." {
251 } else if child := path + "/" + fi.Name(); fi.IsDir() {
252 add, err := allGobFiles(child)
256 files = append(files, add...)
257 } else if strings.HasSuffix(child, ".gob") || strings.HasSuffix(child, ".gob.gz") {
258 files = append(files, child)
264 func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string) error {
265 log.Infof("LoadDir: walk dir %s", path)
266 files, err := allGobFiles(path)
270 ctx, cancel := context.WithCancel(ctx)
273 allcgs := make([][]CompactGenome, len(files))
274 allcseqs := make([][]CompactSequence, len(files))
275 allvariantmap := map[tileLibRef]tileVariantID{}
276 errs := make(chan error, len(files))
277 log.Infof("LoadDir: read %d files", len(files))
278 for fileno, path := range files {
279 fileno, path := fileno, path
287 defer log.Infof("LoadDir: finished reading %s", path)
289 var variantmap = map[tileLibRef]tileVariantID{}
290 var cgs []CompactGenome
291 var cseqs []CompactSequence
292 err = DecodeLibrary(f, strings.HasSuffix(path, ".gz"), func(ent *LibraryEntry) error {
293 if ctx.Err() != nil {
296 if len(ent.TagSet) > 0 {
298 if tilelib.taglib == nil || tilelib.taglib.Len() != len(ent.TagSet) {
299 // load first set of tags, or
300 // report mismatch if 2 sets
301 // have different #tags.
302 if err := tilelib.loadTagSet(ent.TagSet); err != nil {
309 for _, tv := range ent.TileVariants {
310 variantmap[tileLibRef{Tag: tv.Tag, Variant: tv.Variant}] = tilelib.getRef(tv.Tag, tv.Sequence).Variant
312 cgs = append(cgs, ent.CompactGenomes...)
313 cseqs = append(cseqs, ent.CompactSequences...)
317 allcseqs[fileno] = cseqs
320 for k, v := range variantmap {
333 log.Info("LoadDir: loadCompactGenomes")
334 var flatcgs []CompactGenome
335 for _, cgs := range allcgs {
336 flatcgs = append(flatcgs, cgs...)
338 err = tilelib.loadCompactGenomes(flatcgs, allvariantmap)
343 log.Info("LoadDir: loadCompactSequences")
344 var flatcseqs []CompactSequence
345 for _, cseqs := range allcseqs {
346 flatcseqs = append(flatcseqs, cseqs...)
348 err = tilelib.loadCompactSequences(flatcseqs, allvariantmap)
353 log.Info("LoadDir done")
357 func (tilelib *tileLibrary) WriteDir(dir string) error {
359 nfiles := ntilefiles + len(tilelib.refseqs)
360 files := make([]*os.File, nfiles)
361 for i := range files {
362 f, err := os.OpenFile(fmt.Sprintf("%s/library.%04d.gob.gz", dir, i), os.O_CREATE|os.O_WRONLY, 0666)
369 bufws := make([]*bufio.Writer, nfiles)
370 for i := range bufws {
371 bufws[i] = bufio.NewWriterSize(files[i], 1<<26)
373 zws := make([]*pgzip.Writer, nfiles)
375 zws[i] = pgzip.NewWriter(bufws[i])
378 encoders := make([]*gob.Encoder, nfiles)
379 for i := range encoders {
380 encoders[i] = gob.NewEncoder(zws[i])
383 cgnames := make([]string, 0, len(tilelib.compactGenomes))
384 for name := range tilelib.compactGenomes {
385 cgnames = append(cgnames, name)
387 sort.Strings(cgnames)
389 refnames := make([]string, 0, len(tilelib.refseqs))
390 for name := range tilelib.refseqs {
391 refnames = append(refnames, name)
393 sort.Strings(refnames)
395 log.Infof("WriteDir: writing %d files", nfiles)
396 ctx, cancel := context.WithCancel(context.Background())
398 errs := make(chan error, nfiles)
399 for start := range files {
402 err := encoders[start].Encode(LibraryEntry{TagSet: tilelib.taglib.Tags()})
407 if refidx := start - ntilefiles; refidx >= 0 {
408 // write each ref to its own file
409 // (they seem to load very slowly)
410 name := refnames[refidx]
411 errs <- encoders[start].Encode(LibraryEntry{CompactSequences: []CompactSequence{{
413 TileSequences: tilelib.refseqs[name],
417 for i := start; i < len(cgnames); i += ntilefiles {
418 err := encoders[start].Encode(LibraryEntry{CompactGenomes: []CompactGenome{{
420 Variants: tilelib.compactGenomes[cgnames[i]],
427 tvs := []TileVariant{}
428 for tag := start; tag < len(tilelib.variant) && ctx.Err() == nil; tag += ntilefiles {
430 for idx, hash := range tilelib.variant[tag] {
431 tvs = append(tvs, TileVariant{
433 Variant: tileVariantID(idx + 1),
435 Sequence: tilelib.hashSequence(hash),
438 err := encoders[start].Encode(LibraryEntry{TileVariants: tvs})
453 log.Info("WriteDir: flushing")
455 err := zws[i].Close()
459 err = bufws[i].Flush()
463 err = files[i].Close()
468 log.Info("WriteDir: done")
472 // Load library data from rdr. Tile variants might be renumbered in
473 // the process; in that case, genomes variants will be renumbered to
475 func (tilelib *tileLibrary) LoadGob(ctx context.Context, rdr io.Reader, gz bool) error {
476 cgs := []CompactGenome{}
477 cseqs := []CompactSequence{}
478 variantmap := map[tileLibRef]tileVariantID{}
479 err := DecodeLibrary(rdr, gz, func(ent *LibraryEntry) error {
480 if ctx.Err() != nil {
483 if err := tilelib.loadTagSet(ent.TagSet); err != nil {
486 if err := tilelib.loadTileVariants(ent.TileVariants, variantmap); err != nil {
489 cgs = append(cgs, ent.CompactGenomes...)
490 cseqs = append(cseqs, ent.CompactSequences...)
496 if ctx.Err() != nil {
499 err = tilelib.loadCompactGenomes(cgs, variantmap)
503 err = tilelib.loadCompactSequences(cseqs, variantmap)
510 func (tilelib *tileLibrary) dump(out io.Writer) {
511 printTV := func(tag int, variant tileVariantID) {
513 fmt.Fprintf(out, " -")
514 } else if tag >= len(tilelib.variant) {
515 fmt.Fprintf(out, " (!tag=%d)", tag)
516 } else if int(variant) > len(tilelib.variant[tag]) {
517 fmt.Fprintf(out, " (tag=%d,!variant=%d)", tag, variant)
519 fmt.Fprintf(out, " %x", tilelib.variant[tag][variant-1][:8])
522 for refname, refseqs := range tilelib.refseqs {
523 for seqname, seq := range refseqs {
524 fmt.Fprintf(out, "ref %s %s", refname, seqname)
525 for _, libref := range seq {
526 printTV(int(libref.Tag), libref.Variant)
528 fmt.Fprintf(out, "\n")
531 for name, cg := range tilelib.compactGenomes {
532 fmt.Fprintf(out, "cg %s", name)
533 for tag, variant := range cg {
534 printTV(tag/2, variant)
536 fmt.Fprintf(out, "\n")
540 type importStats struct {
546 DroppedOutOfOrderTiles int
549 func (tilelib *tileLibrary) TileFasta(filelabel string, rdr io.Reader, matchChromosome *regexp.Regexp) (tileSeq, []importStats, error) {
555 todo := make(chan jobT, 1)
556 scanner := bufio.NewScanner(rdr)
562 buf := scanner.Bytes()
563 if len(buf) > 0 && buf[0] == '>' {
564 todo <- jobT{seqlabel, append([]byte(nil), fasta...)}
565 seqlabel, fasta = strings.SplitN(string(buf[1:]), " ", 2)[0], fasta[:0]
566 log.Debugf("%s %s reading fasta", filelabel, seqlabel)
568 fasta = append(fasta, bytes.ToLower(buf)...)
571 todo <- jobT{seqlabel, fasta}
573 type foundtag struct {
577 found := make([]foundtag, 2000000)
578 path := make([]tileLibRef, 2000000)
581 skippedSequences := 0
582 taglen := tilelib.taglib.TagLen()
583 var stats []importStats
584 for job := range todo {
585 if len(job.fasta) == 0 {
587 } else if !matchChromosome.MatchString(job.label) {
591 log.Debugf("%s %s tiling", filelabel, job.label)
594 tilelib.taglib.FindAll(job.fasta, func(tagid tagID, pos, taglen int) {
595 found = append(found, foundtag{pos: pos, tagid: tagid})
597 totalFoundTags += len(found)
599 log.Warnf("%s %s no tags found", filelabel, job.label)
604 log.Infof("%s %s keeping longest increasing subsequence", filelabel, job.label)
605 keep := longestIncreasingSubsequence(len(found), func(i int) int { return int(found[i].tagid) })
606 for i, x := range keep {
609 skipped = len(found) - len(keep)
610 found = found[:len(keep)]
613 log.Infof("%s %s getting %d librefs", filelabel, job.label, len(found))
614 throttle := &throttle{Max: runtime.NumCPU()}
615 path = path[:len(found)]
617 for i, f := range found {
621 defer throttle.Release()
622 var startpos, endpos int
628 if i == len(found)-1 {
629 endpos = len(job.fasta)
631 endpos = found[i+1].pos + taglen
633 path[i] = tilelib.getRef(f.tagid, job.fasta[startpos:endpos])
634 if countBases(job.fasta[startpos:endpos]) != endpos-startpos {
635 atomic.AddInt64(&lowquality, 1)
641 log.Infof("%s %s copying path", filelabel, job.label)
643 pathcopy := make([]tileLibRef, len(path))
645 ret[job.label] = pathcopy
647 basesIn := countBases(job.fasta)
648 log.Infof("%s %s fasta in %d coverage in %d path len %d low-quality %d skipped-out-of-order %d", filelabel, job.label, len(job.fasta), basesIn, len(path), lowquality, skipped)
649 stats = append(stats, importStats{
650 InputFile: filelabel,
651 InputLabel: job.label,
652 InputLength: len(job.fasta),
653 InputCoverage: basesIn,
654 PathLength: len(path),
655 DroppedOutOfOrderTiles: skipped,
658 totalPathLen += len(path)
660 log.Printf("%s tiled with total path len %d in %d sequences (skipped %d sequences that did not match chromosome regexp, skipped %d out-of-order tags)", filelabel, totalPathLen, len(ret), skippedSequences, totalFoundTags-totalPathLen)
661 return ret, stats, scanner.Err()
664 func (tilelib *tileLibrary) Len() int64 {
665 return atomic.LoadInt64(&tilelib.variants)
668 // Return a tileLibRef for a tile with the given tag and sequence,
669 // adding the sequence to the library if needed.
670 func (tilelib *tileLibrary) getRef(tag tagID, seq []byte) tileLibRef {
672 if !tilelib.retainNoCalls {
673 for _, b := range seq {
674 if b != 'a' && b != 'c' && b != 'g' && b != 't' {
680 seqhash := blake2b.Sum256(seq)
681 var vlock sync.Locker
684 if len(tilelib.vlock) > int(tag) {
685 vlock = tilelib.vlock[tag]
687 tilelib.mtx.RUnlock()
691 for i, varhash := range tilelib.variant[tag] {
692 if varhash == seqhash {
694 return tileLibRef{Tag: tag, Variant: tileVariantID(i + 1)}
700 if tilelib.variant == nil && tilelib.taglib != nil {
701 tilelib.variant = make([][][blake2b.Size256]byte, tilelib.taglib.Len())
702 tilelib.vlock = make([]sync.Locker, tilelib.taglib.Len())
703 for i := range tilelib.vlock {
704 tilelib.vlock[i] = new(sync.Mutex)
707 if int(tag) >= len(tilelib.variant) {
708 oldlen := len(tilelib.vlock)
709 for i := 0; i < oldlen; i++ {
710 tilelib.vlock[i].Lock()
712 // If we haven't seen the tag library yet (as
713 // in a merge), tilelib.taglib.Len() is
714 // zero. We can still behave correctly, we
715 // just need to expand the tilelib.variant and
716 // tilelib.vlock slices as needed.
717 if int(tag) >= cap(tilelib.variant) {
718 // Allocate 2x capacity.
719 newslice := make([][][blake2b.Size256]byte, int(tag)+1, (int(tag)+1)*2)
720 copy(newslice, tilelib.variant)
721 tilelib.variant = newslice[:int(tag)+1]
722 newvlock := make([]sync.Locker, int(tag)+1, (int(tag)+1)*2)
723 copy(newvlock, tilelib.vlock)
724 tilelib.vlock = newvlock[:int(tag)+1]
726 // Use previously allocated capacity,
728 tilelib.variant = tilelib.variant[:int(tag)+1]
729 tilelib.vlock = tilelib.vlock[:int(tag)+1]
731 for i := oldlen; i < len(tilelib.vlock); i++ {
732 tilelib.vlock[i] = new(sync.Mutex)
734 for i := 0; i < oldlen; i++ {
735 tilelib.vlock[i].Unlock()
738 vlock = tilelib.vlock[tag]
743 for i, varhash := range tilelib.variant[tag] {
744 if varhash == seqhash {
746 return tileLibRef{Tag: tag, Variant: tileVariantID(i + 1)}
749 atomic.AddInt64(&tilelib.variants, 1)
750 tilelib.variant[tag] = append(tilelib.variant[tag], seqhash)
751 variant := tileVariantID(len(tilelib.variant[tag]))
754 if tilelib.retainTileSequences && !dropSeq {
755 seqCopy := append([]byte(nil), seq...)
756 if tilelib.seq2 == nil {
758 if tilelib.seq2 == nil {
759 tilelib.seq2lock = map[[2]byte]sync.Locker{}
760 m := map[[2]byte]map[[blake2b.Size256]byte][]byte{}
762 for i := 0; i < 256; i++ {
764 for j := 0; j < 256; j++ {
766 m[k] = map[[blake2b.Size256]byte][]byte{}
767 tilelib.seq2lock[k] = &sync.Mutex{}
775 copy(k[:], seqhash[:])
776 locker := tilelib.seq2lock[k]
778 tilelib.seq2[k][seqhash] = seqCopy
784 // Save the hash, but not the sequence
787 if tilelib.encoder != nil {
788 tilelib.encoder.Encode(LibraryEntry{
789 TileVariants: []TileVariant{{
797 if tilelib.onAddTileVariant != nil {
798 tilelib.onAddTileVariant(tileLibRef{tag, variant}, seqhash, saveSeq)
800 return tileLibRef{Tag: tag, Variant: variant}
803 func (tilelib *tileLibrary) hashSequence(hash [blake2b.Size256]byte) []byte {
804 var partition [2]byte
805 copy(partition[:], hash[:])
806 return tilelib.seq2[partition][hash]
809 func (tilelib *tileLibrary) TileVariantSequence(libref tileLibRef) []byte {
810 if libref.Variant == 0 || len(tilelib.variant) <= int(libref.Tag) || len(tilelib.variant[libref.Tag]) < int(libref.Variant) {
813 return tilelib.hashSequence(tilelib.variant[libref.Tag][libref.Variant-1])
816 // Tidy deletes unreferenced tile variants and renumbers variants so
817 // more common variants have smaller IDs.
818 func (tilelib *tileLibrary) Tidy() {
819 log.Print("Tidy: compute inref")
820 inref := map[tileLibRef]bool{}
821 for _, refseq := range tilelib.refseqs {
822 for _, librefs := range refseq {
823 for _, libref := range librefs {
828 log.Print("Tidy: compute remap")
829 remap := make([][]tileVariantID, len(tilelib.variant))
830 throttle := throttle{Max: runtime.NumCPU() + 1}
831 for tag, oldvariants := range tilelib.variant {
832 tag, oldvariants := tagID(tag), oldvariants
833 if tag%1000000 == 0 {
834 log.Printf("Tidy: tag %d", tag)
838 defer throttle.Release()
839 uses := make([]int, len(oldvariants))
840 for _, cg := range tilelib.compactGenomes {
841 for phase := 0; phase < 2; phase++ {
842 cgi := int(tag)*2 + phase
843 if cgi < len(cg) && cg[cgi] > 0 {
849 // Compute desired order of variants:
850 // neworder[x] == index in oldvariants that
851 // should move to position x.
852 neworder := make([]int, len(oldvariants))
853 for i := range neworder {
856 sort.Slice(neworder, func(i, j int) bool {
857 if cmp := uses[neworder[i]] - uses[neworder[j]]; cmp != 0 {
860 return bytes.Compare(oldvariants[neworder[i]][:], oldvariants[neworder[j]][:]) < 0
864 // Replace tilelib.variant[tag] with a new
865 // re-ordered slice of hashes, and make a
866 // mapping from old to new variant IDs.
867 remaptag := make([]tileVariantID, len(oldvariants)+1)
868 newvariants := make([][blake2b.Size256]byte, 0, len(neworder))
869 for _, oldi := range neworder {
870 if uses[oldi] > 0 || inref[tileLibRef{Tag: tag, Variant: tileVariantID(oldi + 1)}] {
871 newvariants = append(newvariants, oldvariants[oldi])
872 remaptag[oldi+1] = tileVariantID(len(newvariants))
875 tilelib.variant[tag] = newvariants
876 remap[tag] = remaptag
881 // Apply remap to genomes and reference sequences, so they
882 // refer to the same tile variants using the changed IDs.
883 log.Print("Tidy: apply remap")
884 var wg sync.WaitGroup
885 for _, cg := range tilelib.compactGenomes {
890 for idx, variant := range cg {
891 cg[idx] = remap[tagID(idx/2)][variant]
895 for _, refcs := range tilelib.refseqs {
896 for _, refseq := range refcs {
901 for i, tv := range refseq {
902 refseq[i].Variant = remap[tv.Tag][tv.Variant]
908 log.Print("Tidy: done")
911 func countBases(seq []byte) int {
913 for _, c := range seq {