No issue #
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
retainNoCalls: true,
retainTileSequences: true,
}
retainNoCalls: true,
retainTileSequences: true,
}
- err = tilelib.LoadGob(context.Background(), input, strings.HasSuffix(*inputFilename, ".gz"), nil)
+ err = tilelib.LoadGob(context.Background(), input, strings.HasSuffix(*inputFilename, ".gz"))
if err != nil {
return 1
}
if err != nil {
return 1
}
retainTileSequences: true,
compactGenomes: map[string][]tileVariantID{},
}
retainTileSequences: true,
compactGenomes: map[string][]tileVariantID{},
}
- err = tilelib.LoadDir(context.Background(), *inputDir, nil)
+ err = tilelib.LoadDir(context.Background(), *inputDir)
if err != nil {
return 1
}
if err != nil {
return 1
}
retainTileSequences: true,
compactGenomes: map[string][]tileVariantID{},
}
retainTileSequences: true,
compactGenomes: map[string][]tileVariantID{},
}
- err = tilelib.LoadDir(context.Background(), *inputDir, nil)
+ err = tilelib.LoadDir(context.Background(), *inputDir)
if err != nil {
return 1
}
if err != nil {
return 1
}
retainTileSequences: true,
compactGenomes: map[string][]tileVariantID{},
}
retainTileSequences: true,
compactGenomes: map[string][]tileVariantID{},
}
- err = tilelib.LoadDir(context.Background(), *inputDir, nil)
+ err = tilelib.LoadDir(context.Background(), *inputDir)
if err != nil {
return 1
}
if err != nil {
return 1
}
go func(input string) {
defer wg.Done()
log.Printf("%s: reading", input)
go func(input string) {
defer wg.Done()
log.Printf("%s: reading", input)
- err := cmd.tilelib.LoadGob(ctx, rdr, strings.HasSuffix(input, ".gz"), nil)
+ err := cmd.tilelib.LoadGob(ctx, rdr, strings.HasSuffix(input, ".gz"))
if err != nil {
cmd.setError(fmt.Errorf("%s: load failed: %w", input, err))
cancel()
if err != nil {
cmd.setError(fmt.Errorf("%s: load failed: %w", input, err))
cancel()
retainNoCalls: true,
compactGenomes: map[string][]tileVariantID{},
}
retainNoCalls: true,
compactGenomes: map[string][]tileVariantID{},
}
- err = tilelib.LoadGob(context.Background(), input, strings.HasSuffix(*inputFilename, ".gz"), nil)
+ err = tilelib.LoadGob(context.Background(), input, strings.HasSuffix(*inputFilename, ".gz"))
if err != nil {
return 1
}
if err != nil {
return 1
}
// if non-nil, write out any tile variants added while tiling
encoder *gob.Encoder
// if non-nil, write out any tile variants added while tiling
encoder *gob.Encoder
+ onAddTileVariant func(libref tileLibRef, hash [blake2b.Size256]byte, seq []byte) error
+ onAddGenome func(CompactGenome) error
+ onAddRefseq func(CompactSequence) error
+
mtx sync.RWMutex
vlock []sync.Locker
}
mtx sync.RWMutex
vlock []sync.Locker
}
-func (tilelib *tileLibrary) loadCompactGenomes(cgs []CompactGenome, variantmap map[tileLibRef]tileVariantID, onLoadGenome func(CompactGenome)) error {
+func (tilelib *tileLibrary) loadCompactGenomes(cgs []CompactGenome, variantmap map[tileLibRef]tileVariantID) error {
log.Debugf("loadCompactGenomes: %d", len(cgs))
var wg sync.WaitGroup
errs := make(chan error, 1)
log.Debugf("loadCompactGenomes: %d", len(cgs))
var wg sync.WaitGroup
errs := make(chan error, 1)
// log.Tracef("loadCompactGenomes: cg %s tag %d variant %d => %d", cg.Name, tag, variant, newvariant)
cg.Variants[i] = newvariant
}
// log.Tracef("loadCompactGenomes: cg %s tag %d variant %d => %d", cg.Name, tag, variant, newvariant)
cg.Variants[i] = newvariant
}
- if onLoadGenome != nil {
- onLoadGenome(cg)
+ if tilelib.onAddGenome != nil {
+ err := tilelib.onAddGenome(cg)
+ if err != nil {
+ select {
+ case errs <- err:
+ default:
+ }
+ return
+ }
}
if tilelib.encoder != nil {
err := tilelib.encoder.Encode(LibraryEntry{
}
if tilelib.encoder != nil {
err := tilelib.encoder.Encode(LibraryEntry{
+ if tilelib.onAddRefseq != nil {
+ err := tilelib.onAddRefseq(cseq)
+ if err != nil {
+ return err
+ }
+ }
log.Infof("loadCompactSequences: checking %s done", cseq.Name)
}
tilelib.mtx.Lock()
log.Infof("loadCompactSequences: checking %s done", cseq.Name)
}
tilelib.mtx.Lock()
-func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGenome func(CompactGenome)) error {
+func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string) error {
var files []string
var walk func(string) error
walk = func(path string) error {
var files []string
var walk func(string) error
walk = func(path string) error {
for _, cgs := range allcgs {
flatcgs = append(flatcgs, cgs...)
}
for _, cgs := range allcgs {
flatcgs = append(flatcgs, cgs...)
}
- err = tilelib.loadCompactGenomes(flatcgs, allvariantmap, onLoadGenome)
+ err = tilelib.loadCompactGenomes(flatcgs, allvariantmap)
if err != nil {
return err
}
if err != nil {
return err
}
// Load library data from rdr. Tile variants might be renumbered in
// the process; in that case, genomes variants will be renumbered to
// match.
// Load library data from rdr. Tile variants might be renumbered in
// the process; in that case, genomes variants will be renumbered to
// match.
-//
-// If onLoadGenome is non-nil, call it on each CompactGenome entry.
-func (tilelib *tileLibrary) LoadGob(ctx context.Context, rdr io.Reader, gz bool, onLoadGenome func(CompactGenome)) error {
+func (tilelib *tileLibrary) LoadGob(ctx context.Context, rdr io.Reader, gz bool) error {
cgs := []CompactGenome{}
cseqs := []CompactSequence{}
variantmap := map[tileLibRef]tileVariantID{}
cgs := []CompactGenome{}
cseqs := []CompactSequence{}
variantmap := map[tileLibRef]tileVariantID{}
if ctx.Err() != nil {
return ctx.Err()
}
if ctx.Err() != nil {
return ctx.Err()
}
- err = tilelib.loadCompactGenomes(cgs, variantmap, onLoadGenome)
+ err = tilelib.loadCompactGenomes(cgs, variantmap)
if err != nil {
return err
}
if err != nil {
return err
}
+ saveSeq := seq
+ if dropSeq {
+ // Save the hash, but not the sequence
+ saveSeq = nil
+ }
if tilelib.encoder != nil {
if tilelib.encoder != nil {
- saveSeq := seq
- if dropSeq {
- // Save the hash, but not the sequence
- saveSeq = nil
- }
tilelib.encoder.Encode(LibraryEntry{
TileVariants: []TileVariant{{
Tag: tag,
tilelib.encoder.Encode(LibraryEntry{
TileVariants: []TileVariant{{
Tag: tag,
+ if tilelib.onAddTileVariant != nil {
+ tilelib.onAddTileVariant(tileLibRef{tag, variant}, seqhash, saveSeq)
+ }
return tileLibRef{Tag: tag, Variant: variant}
}
return tileLibRef{Tag: tag, Variant: variant}
}