19 "git.arvados.org/arvados.git/sdk/go/arvados"
20 "github.com/arvados/lightning/hgvs"
21 log "github.com/sirupsen/logrus"
24 type outputFormat struct {
25 Print func(out io.Writer, seqname string, varslice []hgvs.Variant)
30 outputFormats = map[string]outputFormat{
31 "hgvs": outputFormatHGVS,
32 "vcf": outputFormatVCF,
34 outputFormatHGVS = outputFormat{Print: printHGVS}
35 outputFormatVCF = outputFormat{Print: printVCF, PadLeft: true}
38 type exporter struct {
39 outputFormat outputFormat
42 func (cmd *exporter) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
46 fmt.Fprintf(stderr, "%s\n", err)
49 flags := flag.NewFlagSet("", flag.ContinueOnError)
50 flags.SetOutput(stderr)
51 pprof := flags.String("pprof", "", "serve Go profile data at http://`[addr]:port`")
52 runlocal := flags.Bool("local", false, "run on local host (default: run in an arvados container)")
53 projectUUID := flags.String("project", "", "project `UUID` for output data")
54 priority := flags.Int("priority", 500, "container request priority")
55 refname := flags.String("ref", "", "reference genome `name`")
56 inputFilename := flags.String("i", "-", "input `file` (library)")
57 outputFilename := flags.String("o", "-", "output `file`")
58 outputFormatStr := flags.String("output-format", "hgvs", "output `format`: hgvs or vcf")
59 outputBed := flags.String("output-bed", "", "also output bed `file`")
60 pick := flags.String("pick", "", "`name` of single genome to export")
61 err = flags.Parse(args)
62 if err == flag.ErrHelp {
65 } else if err != nil {
69 if f, ok := outputFormats[*outputFormatStr]; !ok {
70 err = fmt.Errorf("invalid output format %q", *outputFormatStr)
78 log.Println(http.ListenAndServe(*pprof, nil))
83 if *outputFilename != "-" {
84 err = errors.New("cannot specify output file in container mode: not implemented")
87 runner := arvadosContainerRunner{
88 Name: "lightning export",
89 Client: arvados.NewClientFromEnv(),
90 ProjectUUID: *projectUUID,
95 err = runner.TranslatePaths(inputFilename)
100 if strings.Contains(*outputBed, "/") {
101 err = fmt.Errorf("cannot use -output-bed filename %q containing '/' char", *outputBed)
104 *outputBed = "/mnt/output/" + *outputBed
106 runner.Args = []string{"export", "-local=true", "-pick", *pick, "-ref", *refname, "-output-format", *outputFormatStr, "-output-bed", *outputBed, "-i", *inputFilename, "-o", "/mnt/output/export.csv"}
108 output, err = runner.Run()
112 fmt.Fprintln(stdout, output+"/export.csv")
116 input, err := os.Open(*inputFilename)
122 // Error out early if seeking doesn't work on the input file.
123 _, err = input.Seek(0, io.SeekEnd)
127 _, err = input.Seek(0, io.SeekStart)
133 var cgs []CompactGenome
134 var tilelib tileLibrary
135 err = tilelib.LoadGob(context.Background(), input, func(cg CompactGenome) {
136 if *pick != "" && *pick != cg.Name {
139 log.Debugf("export: pick %q", cg.Name)
142 cgs = append(cgs, cg)
147 sort.Slice(cgs, func(i, j int) bool { return cgs[i].Name < cgs[j].Name })
148 log.Printf("export: pick %q => %d genomes", *pick, len(cgs))
150 refseq, ok := tilelib.refseqs[*refname]
152 err = fmt.Errorf("reference name %q not found in input; have %v", *refname, func() (names []string) {
153 for name := range tilelib.refseqs {
154 names = append(names, name)
161 _, err = input.Seek(0, io.SeekStart)
166 var output io.WriteCloser
167 if *outputFilename == "-" {
168 output = nopCloser{stdout}
170 output, err = os.OpenFile(*outputFilename, os.O_CREATE|os.O_WRONLY, 0666)
176 bufw := bufio.NewWriter(output)
179 var bedbufw *bufio.Writer
180 if *outputBed != "" {
181 bedout, err = os.OpenFile(*outputBed, os.O_CREATE|os.O_WRONLY, 0666)
186 bedbufw = bufio.NewWriter(bedout)
189 err = cmd.export(bufw, bedout, input, tilelib.taglib.keylen, refseq, cgs)
202 err = bedbufw.Flush()
218 func (cmd *exporter) export(out, bedout io.Writer, librdr io.Reader, taglen int, refseq map[string][]tileLibRef, cgs []CompactGenome) error {
219 need := map[tileLibRef]bool{}
220 var seqnames []string
221 for seqname, librefs := range refseq {
222 seqnames = append(seqnames, seqname)
223 for _, libref := range librefs {
224 if libref.Variant != 0 {
229 sort.Strings(seqnames)
231 for _, cg := range cgs {
232 for i, variant := range cg.Variants {
236 libref := tileLibRef{Tag: tagID(i / 2), Variant: variant}
241 log.Infof("export: loading %d tile variants", len(need))
242 tileVariant := map[tileLibRef]TileVariant{}
243 err := DecodeLibrary(librdr, func(ent *LibraryEntry) error {
244 for _, tv := range ent.TileVariants {
245 libref := tileLibRef{Tag: tv.Tag, Variant: tv.Variant}
247 tileVariant[libref] = tv
256 log.Infof("export: loaded %d tile variants", len(tileVariant))
257 var missing []tileLibRef
258 for libref := range need {
259 if _, ok := tileVariant[libref]; !ok {
260 missing = append(missing, libref)
263 if len(missing) > 0 {
264 if limit := 100; len(missing) > limit {
265 log.Warnf("first %d missing tiles: %v", limit, missing[:limit])
267 log.Warnf("missing tiles: %v", missing)
269 return fmt.Errorf("%d needed tiles are missing from library", len(missing))
273 for _, seqname := range seqnames {
274 variantAt := map[int][]hgvs.Variant{} // variantAt[chromOffset][genomeIndex*2+phase]
275 for refstep, libref := range refseq[seqname] {
276 reftile := tileVariant[libref]
277 coverage := int64(0) // number of ref bases that are called in genomes -- max is len(reftile.Sequence)*len(cgs)*2
278 for cgidx, cg := range cgs {
279 for phase := 0; phase < 2; phase++ {
280 if len(cg.Variants) <= int(libref.Tag)*2+phase {
283 variant := cg.Variants[int(libref.Tag)*2+phase]
287 genometile := tileVariant[tileLibRef{Tag: libref.Tag, Variant: variant}]
288 if variant == libref.Variant {
291 refSequence := reftile.Sequence
292 // If needed, extend the
293 // reference sequence up to
294 // the tag at the end of the
295 // genometile sequence.
296 refstepend := refstep + 1
297 for refstepend < len(refseq[seqname]) && len(refSequence) >= taglen && !bytes.EqualFold(refSequence[len(refSequence)-taglen:], genometile.Sequence[len(genometile.Sequence)-taglen:]) {
298 if &refSequence[0] == &reftile.Sequence[0] {
299 refSequence = append([]byte(nil), refSequence...)
301 refSequence = append(refSequence, tileVariant[refseq[seqname][refstepend]].Sequence...)
304 // (TODO: handle no-calls)
305 vars, _ := hgvs.Diff(strings.ToUpper(string(refSequence)), strings.ToUpper(string(genometile.Sequence)), time.Second)
306 for _, v := range vars {
307 if cmd.outputFormat.PadLeft {
311 log.Debugf("%s seq %s phase %d tag %d tile diff %s\n", cg.Name, seqname, phase, libref.Tag, v.String())
312 varslice := variantAt[v.Position]
314 varslice = make([]hgvs.Variant, len(cgs)*2)
315 variantAt[v.Position] = varslice
317 varslice[cgidx*2+phase] = v
319 coverage += int64(len(reftile.Sequence))
322 refpos += len(reftile.Sequence) - taglen
324 // Flush entries from variantAt that are
325 // behind refpos. Flush all entries if this is
326 // the last reftile of the path/chromosome.
328 lastrefstep := refstep == len(refseq[seqname])-1
329 for pos := range variantAt {
330 if lastrefstep || pos <= refpos {
331 flushpos = append(flushpos, pos)
334 sort.Slice(flushpos, func(i, j int) bool { return flushpos[i] < flushpos[j] })
335 for _, pos := range flushpos {
336 varslice := variantAt[pos]
337 delete(variantAt, pos)
338 for i := range varslice {
339 if varslice[i].Position == 0 {
340 varslice[i].Position = pos
343 cmd.outputFormat.Print(out, seqname, varslice)
345 if bedout != nil && len(reftile.Sequence) > 0 {
346 tilestart := refpos - len(reftile.Sequence) + taglen
351 thickstart := tilestart + taglen
356 // coverage score, 0 to 1000
357 score := 1000 * coverage / int64(len(reftile.Sequence)) / int64(len(cgs)) / 2
358 fmt.Fprintf(bedout, "%s %d %d %d %d . %d %d\n",
359 seqname, tilestart, tileend,
362 thickstart, thickend)
369 func printVCF(out io.Writer, seqname string, varslice []hgvs.Variant) {
370 refs := map[string]map[string]int{}
371 for _, v := range varslice {
372 if v.Ref == "" && v.New == "" {
377 alts = map[string]int{}
382 for ref, alts := range refs {
383 var altslice []string
384 for alt := range alts {
385 altslice = append(altslice, alt)
387 sort.Strings(altslice)
388 for i, a := range altslice {
391 fmt.Fprintf(out, "%s\t%d\t%s\t%s", seqname, varslice[0].Position, ref, strings.Join(altslice, ","))
392 for i := 0; i < len(varslice)/2; i++ {
393 v1, v2 := varslice[i*2], varslice[i*2+1]
394 a1, a2 := alts[v1.New], alts[v2.New]
401 fmt.Fprintf(out, "\t%d/%d", a1, a2)
403 out.Write([]byte{'\n'})
407 func printHGVS(out io.Writer, seqname string, varslice []hgvs.Variant) {
408 for i := 0; i < len(varslice)/2; i++ {
410 out.Write([]byte{'\t'})
412 var1, var2 := varslice[i*2], varslice[i*2+1]
414 if var1.Ref == var1.New {
415 out.Write([]byte{'.'})
417 fmt.Fprintf(out, "%s:g.%s", seqname, var1.String())
420 fmt.Fprintf(out, "%s:g.[%s];[%s]", seqname, var1.String(), var2.String())
423 out.Write([]byte{'\n'})