type TileVariant struct {
Tag tagID
+ Ref bool
Variant tileVariantID
Blake2b [blake2b.Size256]byte
Sequence []byte
return nil
}
-func (cmd *importer) tileFasta(tilelib *tileLibrary, infile string) (tileSeq, []importStats, error) {
+func (cmd *importer) tileFasta(tilelib *tileLibrary, infile string, isRef bool) (tileSeq, []importStats, error) {
var input io.ReadCloser
input, err := open(infile)
if err != nil {
}
defer input.Close()
}
- return tilelib.TileFasta(infile, input, cmd.matchChromosome)
+ return tilelib.TileFasta(infile, input, cmd.matchChromosome, isRef)
}
func (cmd *importer) loadTagLibrary() (*tagLibrary, error) {
defer phases.Done()
log.Printf("%s starting", infile)
defer log.Printf("%s done", infile)
- tseqs, stats, err := cmd.tileFasta(tilelib, infile)
+ tseqs, stats, err := cmd.tileFasta(tilelib, infile, false)
allstats[idx*2] = stats
var kept, dropped int
variants[0], kept, dropped = tseqs.Variants()
defer phases.Done()
log.Printf("%s starting", infile2)
defer log.Printf("%s done", infile2)
- tseqs, stats, err := cmd.tileFasta(tilelib, infile2)
+ tseqs, stats, err := cmd.tileFasta(tilelib, infile2, false)
allstats[idx*2+1] = stats
var kept, dropped int
variants[1], kept, dropped = tseqs.Variants()
defer phases.Done()
log.Printf("%s starting", infile)
defer log.Printf("%s done", infile)
- tseqs, stats, err := cmd.tileFasta(tilelib, infile)
+ tseqs, stats, err := cmd.tileFasta(tilelib, infile, true)
allstats[idx*2] = stats
if err != nil {
return err
return
}
defer consensus.Wait()
- tileseq, stats, err = tilelib.TileFasta(fmt.Sprintf("%s phase %d", infile, phase+1), stdout, cmd.matchChromosome)
+ tileseq, stats, err = tilelib.TileFasta(fmt.Sprintf("%s phase %d", infile, phase+1), stdout, cmd.matchChromosome, false)
if err != nil {
return
}
atomic.AddInt64(&countTileVariants, int64(len(ent.TileVariants)))
for _, tv := range ent.TileVariants {
tv.Variant = tileVariantID(int(tv.Variant)*namespaces + namespace)
- err := encs[int(tv.Tag)/tagsPerFile].Encode(LibraryEntry{
+ fileno := 0
+ if !tv.Ref {
+ fileno = int(tv.Tag) / tagsPerFile
+ }
+ err := encs[fileno].Encode(LibraryEntry{
TileVariants: []TileVariant{tv},
})
if err != nil {
var cgnames []string
var refseq map[string][]tileLibRef
+ var reftiledata = make(map[tileLibRef][]byte, 11000000)
in0, err := open(infiles[0])
if err != nil {
return 1
}
+
taglen := -1
DecodeLibrary(in0, strings.HasSuffix(infiles[0], ".gz"), func(ent *LibraryEntry) error {
if len(ent.TagSet) > 0 {
for _, cg := range ent.CompactGenomes {
cgnames = append(cgnames, cg.Name)
}
+ for _, tv := range ent.TileVariants {
+ if tv.Ref {
+ reftiledata[tileLibRef{tv.Tag, tv.Variant}] = tv.Sequence
+ }
+ }
return nil
})
if err != nil {
}
}
- log.Info("building list of reference tiles to load") // TODO: more efficient if we had saved all ref tiles in slice0
+ log.Info("indexing reference tiles")
type reftileinfo struct {
variant tileVariantID
seqname string // chr1
reftile := map[tagID]*reftileinfo{}
for seqname, cseq := range refseq {
for _, libref := range cseq {
- reftile[libref.Tag] = &reftileinfo{seqname: seqname, variant: libref.Variant}
- }
- }
- log.Info("loading reference tiles from all slices")
- throttleCPU := throttle{Max: runtime.GOMAXPROCS(0)}
- for _, infile := range infiles {
- infile := infile
- throttleCPU.Go(func() error {
- defer log.Infof("%s: done", infile)
- f, err := open(infile)
- if err != nil {
- return err
+ reftile[libref.Tag] = &reftileinfo{
+ seqname: seqname,
+ variant: libref.Variant,
+ tiledata: reftiledata[libref],
}
- defer f.Close()
- return DecodeLibrary(f, strings.HasSuffix(infile, ".gz"), func(ent *LibraryEntry) error {
- for _, tv := range ent.TileVariants {
- if dst, ok := reftile[tv.Tag]; ok && dst.variant == tv.Variant {
- dst.tiledata = tv.Sequence
- }
- }
- return nil
- })
- })
- }
- if err = throttleCPU.Wait(); err != nil {
- return 1
+ }
}
+ throttleCPU := throttle{Max: runtime.GOMAXPROCS(0)}
log.Info("reconstructing reference sequences")
for seqname, cseq := range refseq {
seqname, cseq := seqname, cseq
for _, libref := range cseq {
rt := reftile[libref.Tag]
rt.pos = pos
+ if len(rt.tiledata) == 0 {
+ return fmt.Errorf("missing tiledata for tag %d variant %d in %s in ref", libref.Tag, libref.Variant, seqname)
+ }
pos += len(rt.tiledata) - taglen
}
return nil
log.Infof("%04d: reading %s", infileIdx, infile)
err = DecodeLibrary(f, strings.HasSuffix(infile, ".gz"), func(ent *LibraryEntry) error {
for _, tv := range ent.TileVariants {
+ if tv.Ref {
+ continue
+ }
variants := seq[tv.Tag]
if len(variants) == 0 {
variants = make([]TileVariant, 100)
variants int64
// if non-nil, write out any tile variants added while tiling
encoder *gob.Encoder
+ // set Ref flag when writing new variants to encoder
+ encodeRef bool
onAddTileVariant func(libref tileLibRef, hash [blake2b.Size256]byte, seq []byte) error
onAddGenome func(CompactGenome) error
for _, tv := range tvs {
// Assign a new variant ID (unique across all inputs)
// for each input variant.
- variantmap[tileLibRef{Tag: tv.Tag, Variant: tv.Variant}] = tilelib.getRef(tv.Tag, tv.Sequence).Variant
+ variantmap[tileLibRef{Tag: tv.Tag, Variant: tv.Variant}] = tilelib.getRef(tv.Tag, tv.Sequence, tv.Ref).Variant
}
return nil
}
mtx.Unlock()
}
for _, tv := range ent.TileVariants {
- variantmap[tileLibRef{Tag: tv.Tag, Variant: tv.Variant}] = tilelib.getRef(tv.Tag, tv.Sequence).Variant
+ variantmap[tileLibRef{Tag: tv.Tag, Variant: tv.Variant}] = tilelib.getRef(tv.Tag, tv.Sequence, tv.Ref).Variant
}
cgs = append(cgs, ent.CompactGenomes...)
cseqs = append(cseqs, ent.CompactSequences...)
DroppedOutOfOrderTiles int
}
-func (tilelib *tileLibrary) TileFasta(filelabel string, rdr io.Reader, matchChromosome *regexp.Regexp) (tileSeq, []importStats, error) {
+func (tilelib *tileLibrary) TileFasta(filelabel string, rdr io.Reader, matchChromosome *regexp.Regexp, isRef bool) (tileSeq, []importStats, error) {
ret := tileSeq{}
type jobT struct {
label string
} else {
endpos = found[i+1].pos + taglen
}
- path[i] = tilelib.getRef(f.tagid, job.fasta[startpos:endpos])
+ path[i] = tilelib.getRef(f.tagid, job.fasta[startpos:endpos], isRef)
if countBases(job.fasta[startpos:endpos]) != endpos-startpos {
atomic.AddInt64(&lowquality, 1)
}
// Return a tileLibRef for a tile with the given tag and sequence,
// adding the sequence to the library if needed.
-func (tilelib *tileLibrary) getRef(tag tagID, seq []byte) tileLibRef {
+func (tilelib *tileLibrary) getRef(tag tagID, seq []byte, usedByRef bool) tileLibRef {
dropSeq := false
if !tilelib.retainNoCalls {
for _, b := range seq {
tilelib.encoder.Encode(LibraryEntry{
TileVariants: []TileVariant{{
Tag: tag,
+ Ref: usedByRef,
Variant: variant,
Blake2b: seqhash,
Sequence: saveSeq,
s.tag[0]+
"cccccccccccccccccccc\n"+
s.tag[2]+
- "\n"), matchAllChromosomes)
+ "\n"), matchAllChromosomes, false)
c.Assert(err, check.IsNil)
c.Check(tseq, check.DeepEquals, tileSeq{"test-seq": []tileLibRef{{4, 1}, {0, 1}, {2, 1}}})
s.tag[1]+
"ggggggggggggggggggggggg\n"+
s.tag[2]+
- "\n"), matchAllChromosomes)
+ "\n"), matchAllChromosomes, false)
c.Assert(err, check.IsNil)
c.Check(tseq, check.DeepEquals, tileSeq{"test-seq": []tileLibRef{{0, 1}, {1, 1}, {2, 1}}})
s.tag[3]+
"ggggggggggggggggggggggg\n"+
s.tag[4]+
- "\n"), matchAllChromosomes)
+ "\n"), matchAllChromosomes, false)
c.Assert(err, check.IsNil)
c.Check(tseq, check.DeepEquals, tileSeq{"test-seq": []tileLibRef{{2, 1}, {3, 1}, {4, 1}}})
s.tag[0]+
"ggggggggggggggggggggggg\n"+
s.tag[2]+
- "\n"), matchAllChromosomes)
+ "\n"), matchAllChromosomes, false)
c.Assert(err, check.IsNil)
c.Check(tseq, check.DeepEquals, tileSeq{"test-seq": []tileLibRef{{0, 1}, {2, 1}}})
s.tag[2]+
"ggggggggggggggggggggggg\n"+
s.tag[1]+
- "\n"), matchAllChromosomes)
+ "\n"), matchAllChromosomes, false)
c.Assert(err, check.IsNil)
c.Check(tseq, check.DeepEquals, tileSeq{"test-seq": []tileLibRef{{0, 1}, {1, 1}}})
s.tag[1]+
"ggggggggggggggggggggggg\n"+
s.tag[2]+
- "\n"), matchAllChromosomes)
+ "\n"), matchAllChromosomes, false)
c.Assert(err, check.IsNil)
c.Check(tseq, check.DeepEquals, tileSeq{"test-seq": []tileLibRef{{0, 1}, {1, 1}, {2, 1}}})
s.tag[0]+
"ggggggggggggggggggggggg\n"+
s.tag[4]+
- "\n"), matchAllChromosomes)
+ "\n"), matchAllChromosomes, false)
c.Assert(err, check.IsNil)
c.Check(tseq, check.DeepEquals, tileSeq{"test-seq": []tileLibRef{{0, 1}, {1, 1}, {3, 1}, {4, 1}}})
s.tag[1]+
"ggggggggggggggggggggggg\n"+
s.tag[3]+
- "\n"), matchAllChromosomes)
+ "\n"), matchAllChromosomes, false)
c.Assert(err, check.IsNil)
c.Check(tseq, check.DeepEquals, tileSeq{"test-seq": []tileLibRef{{0, 1}, {1, 1}, {3, 1}}})
}