outputFile string
projectUUID string
runLocal bool
+ skipOOO bool
+ outputTiles bool
+ includeNoCalls bool
encoder *gob.Encoder
}
flags.StringVar(&cmd.outputFile, "o", "-", "output `file`")
flags.StringVar(&cmd.projectUUID, "project", "", "project `UUID` for output data")
flags.BoolVar(&cmd.runLocal, "local", false, "run on local host (default: run in an arvados container)")
+ flags.BoolVar(&cmd.skipOOO, "skip-ooo", false, "skip out-of-order tags")
+ flags.BoolVar(&cmd.outputTiles, "output-tiles", false, "include tile variant sequences in output file")
+ flags.BoolVar(&cmd.includeNoCalls, "include-no-calls", false, "treat tiles with no-calls as regular tiles")
priority := flags.Int("priority", 500, "container request priority")
pprof := flags.String("pprof", "", "serve Go profile data at http://`[addr]:port`")
+ loglevel := flags.String("loglevel", "info", "logging threshold (trace, debug, info, warn, error, fatal, or panic)")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
}()
}
+ lvl, err := log.ParseLevel(*loglevel)
+ if err != nil {
+ return 2
+ }
+ log.SetLevel(lvl)
+
if !cmd.runLocal {
runner := arvadosContainerRunner{
Name: "lightning import",
Client: arvados.NewClientFromEnv(),
ProjectUUID: cmd.projectUUID,
- RAM: 30000000000,
+ RAM: 60000000000,
VCPUs: 16,
Priority: *priority,
}
err = errors.New("cannot specify output file in container mode: not implemented")
return 1
}
- runner.Args = append([]string{"import", "-local=true", "-tag-library", cmd.tagLibraryFile, "-ref", cmd.refFile, "-o", cmd.outputFile}, inputs...)
+ runner.Args = append([]string{"import",
+ "-local=true",
+ "-loglevel=" + *loglevel,
+ fmt.Sprintf("-skip-ooo=%v", cmd.skipOOO),
+ fmt.Sprintf("-output-tiles=%v", cmd.outputTiles),
+ fmt.Sprintf("-include-no-calls=%v", cmd.includeNoCalls),
+ "-tag-library", cmd.tagLibraryFile,
+ "-ref", cmd.refFile,
+ "-o", cmd.outputFile,
+ }, inputs...)
var output string
output, err = runner.Run()
if err != nil {
return 1
}
- tilelib, err := cmd.loadTileLibrary()
+ taglib, err := cmd.loadTagLibrary()
if err != nil {
return 1
}
- go func() {
- for range time.Tick(10 * time.Minute) {
- log.Printf("tilelib.Len() == %d", tilelib.Len())
- }
- }()
var output io.WriteCloser
if cmd.outputFile == "-" {
bufw := bufio.NewWriter(output)
cmd.encoder = gob.NewEncoder(bufw)
+ tilelib := &tileLibrary{taglib: taglib, includeNoCalls: cmd.includeNoCalls, skipOOO: cmd.skipOOO}
+ if cmd.outputTiles {
+ tilelib.encoder = cmd.encoder
+ }
+ go func() {
+ for range time.Tick(10 * time.Minute) {
+ log.Printf("tilelib.Len() == %d", tilelib.Len())
+ }
+ }()
+
err = cmd.tileInputs(tilelib, infiles)
if err != nil {
return 1
return tilelib.TileFasta(infile, input)
}
-func (cmd *importer) loadTileLibrary() (*tileLibrary, error) {
+func (cmd *importer) loadTagLibrary() (*tagLibrary, error) {
log.Printf("tag library %s load starting", cmd.tagLibraryFile)
f, err := os.Open(cmd.tagLibraryFile)
if err != nil {
return nil, fmt.Errorf("cannot tile: tag library is empty")
}
log.Printf("tag library %s load done", cmd.tagLibraryFile)
- return &tileLibrary{taglib: &taglib}, nil
+ return &taglib, nil
}
func listInputFiles(paths []string) (files []string, err error) {
log.Printf("%s starting", infile)
defer log.Printf("%s done", infile)
tseqs, err := cmd.tileFasta(tilelib, infile)
- variants[0] = tseqs.Variants()
+ var kept, dropped int
+ variants[0], kept, dropped = tseqs.Variants()
+ log.Printf("%s found %d unique tags plus %d repeats", infile, kept, dropped)
return err
}
infile2 := regexp.MustCompile(`\.1\.fasta(\.gz)?$`).ReplaceAllString(infile, `.2.fasta$1`)
log.Printf("%s starting", infile2)
defer log.Printf("%s done", infile2)
tseqs, err := cmd.tileFasta(tilelib, infile2)
- variants[1] = tseqs.Variants()
+ var kept, dropped int
+ variants[1], kept, dropped = tseqs.Variants()
+ log.Printf("%s found %d unique tags plus %d repeats", infile, kept, dropped)
return err
}
} else {
log.Printf("%s phase %d starting", infile, phase+1)
defer log.Printf("%s phase %d done", infile, phase+1)
tseqs, err := cmd.tileGVCF(tilelib, infile, phase)
- variants[phase] = tseqs.Variants()
+ var kept, dropped int
+ variants[phase], kept, dropped = tseqs.Variants()
+ log.Printf("%s phase %d found %d unique tags plus %d repeats", infile, phase+1, kept, dropped)
return err
}
}
}
flat := make([]tileVariantID, ntags*2)
for i := 0; i < ntags; i++ {
- flat[i*2] = variants[0][i]
- flat[i*2+1] = variants[1][i]
+ for hap := 0; hap < 2; hap++ {
+ if i < len(variants[hap]) {
+ flat[i*2+hap] = variants[hap][i]
+ }
+ }
}
err := cmd.encoder.Encode(LibraryEntry{
CompactGenomes: []CompactGenome{{Name: infile, Variants: flat}},