const annotationMaxTileSpan = 100
type sliceNumpy struct {
- filter filter
- threads int
- chi2Cases []bool
- chi2PValue float64
- pcaComponents int
- minCoverage int
- includeVariant1 bool
- debugTag tagID
+ filter filter
+ threads int
+ chi2Cases []bool
+ chi2PValue float64
+ pvalueMinFrequency float64
+ pcaComponents int
+ minCoverage int
+ includeVariant1 bool
+ debugTag tagID
cgnames []string
samples []sampleInfo
debugTag := flags.Int("debug-tag", -1, "log debugging details about specified tag")
flags.IntVar(&cmd.threads, "threads", 16, "number of memory-hungry assembly threads, and number of VCPUs to request for arvados container")
flags.Float64Var(&cmd.chi2PValue, "chi2-p-value", 1, "do Χ² test (or logistic regression if -samples file has PCA components) and omit columns with p-value above this threshold")
+ flags.Float64Var(&cmd.pvalueMinFrequency, "pvalue-min-frequency", 0.01, "skip p-value calculation on tile variants below this frequency in the training set")
flags.BoolVar(&cmd.includeVariant1, "include-variant-1", false, "include most common variant when building one-hot matrix")
cmd.filter.Flags(flags)
err := flags.Parse(args)
"-pca-components=" + fmt.Sprintf("%d", cmd.pcaComponents),
"-max-pca-tiles=" + fmt.Sprintf("%d", *maxPCATiles),
"-chi2-p-value=" + fmt.Sprintf("%f", cmd.chi2PValue),
+ "-pvalue-min-frequency=" + fmt.Sprintf("%f", cmd.pvalueMinFrequency),
"-include-variant-1=" + fmt.Sprintf("%v", cmd.includeVariant1),
"-debug-tag=" + fmt.Sprintf("%d", cmd.debugTag),
}
cgnamemap[name] = true
}
- {
- samplesOutFilename := *outputDir + "/samples.csv"
- log.Infof("writing sample metadata to %s", samplesOutFilename)
- var f *os.File
- f, err = os.Create(samplesOutFilename)
- if err != nil {
- return err
- }
- defer f.Close()
- for i, si := range cmd.samples {
- var cc, tv string
- if si.isCase {
- cc = "1"
- } else if si.isControl {
- cc = "0"
- }
- if si.isTraining {
- tv = "1"
- } else {
- tv = "0"
- }
- _, err = fmt.Fprintf(f, "%d,%s,%s,%s\n", i, si.id, cc, tv)
- if err != nil {
- err = fmt.Errorf("write %s: %w", samplesOutFilename, err)
- return err
- }
- }
- err = f.Close()
- if err != nil {
- err = fmt.Errorf("close %s: %w", samplesOutFilename, err)
- return err
- }
- log.Print("done")
+ err = writeSampleInfo(cmd.samples, *outputDir)
+ if err != nil {
+ return err
}
log.Info("indexing reference tiles")
}
log.Print("done")
- samplesOutFilename := *outputDir + "/samples.csv"
- log.Infof("writing sample metadata to %s", samplesOutFilename)
- var f *os.File
- f, err = os.Create(samplesOutFilename)
- if err != nil {
- return err
- }
- defer f.Close()
- pcaLabels := ""
- for i := 0; i < outcols; i++ {
- pcaLabels += fmt.Sprintf(",PCA%d", i)
- }
- _, err = fmt.Fprintf(f, "Index,SampleID,CaseControl,TrainingValidation%s\n", pcaLabels)
- if err != nil {
- return err
- }
- for i, si := range cmd.samples {
- var cc, tv string
- if si.isCase {
- cc = "1"
- } else if si.isControl {
- cc = "0"
- }
- if si.isTraining {
- tv = "1"
- } else if si.isValidation {
- tv = "0"
- }
- var pcavals string
+ log.Print("copying pca components to sampleInfo")
+ for i := range cmd.samples {
+ cmd.samples[i].pcaComponents = make([]float64, outcols)
for c := 0; c < outcols; c++ {
- pcavals += fmt.Sprintf(",%f", pca.At(i, c))
- }
- _, err = fmt.Fprintf(f, "%d,%s,%s,%s%s\n", i, si.id, cc, tv, pcavals)
- if err != nil {
- err = fmt.Errorf("write %s: %w", samplesOutFilename, err)
- return err
+ cmd.samples[i].pcaComponents[i] = pca.At(i, c)
}
}
- err = f.Close()
+ log.Print("done")
+
+ err = writeSampleInfo(cmd.samples, *outputDir)
if err != nil {
- err = fmt.Errorf("close %s: %w", samplesOutFilename, err)
return err
}
- log.Print("done")
}
}
if !*mergeOutput && !*onehotChunked && !*onehotSingle && !*onlyPCA {
isCase: split[2] == "1",
isControl: split[2] == "0",
isTraining: split[3] == "1",
- isValidation: split[3] == "0",
+ isValidation: split[3] == "0" && len(split[2]) > 0, // fix errant 0s in input
pcaComponents: pcaComponents,
})
}
return si, nil
}
+func writeSampleInfo(samples []sampleInfo, outputDir string) error {
+ fnm := outputDir + "/samples.csv"
+ log.Infof("writing sample metadata to %s", fnm)
+ f, err := os.Create(fnm)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ pcaLabels := ""
+ if len(samples) > 0 {
+ for i := range samples[0].pcaComponents {
+ pcaLabels += fmt.Sprintf(",PCA%d", i)
+ }
+ }
+ _, err = fmt.Fprintf(f, "Index,SampleID,CaseControl,TrainingValidation%s\n", pcaLabels)
+ if err != nil {
+ return err
+ }
+ for i, si := range samples {
+ var cc, tv string
+ if si.isCase {
+ cc = "1"
+ } else if si.isControl {
+ cc = "0"
+ }
+ if si.isTraining {
+ tv = "1"
+ } else if si.isValidation {
+ tv = "0"
+ }
+ var pcavals string
+ for _, pcaval := range si.pcaComponents {
+ pcavals += fmt.Sprintf(",%f", pcaval)
+ }
+ _, err = fmt.Fprintf(f, "%d,%s,%s,%s%s\n", i, si.id, cc, tv, pcavals)
+ if err != nil {
+ return fmt.Errorf("write %s: %w", fnm, err)
+ }
+ }
+ err = f.Close()
+ if err != nil {
+ return fmt.Errorf("close %s: %w", fnm, err)
+ }
+ log.Print("done")
+ return nil
+}
+
func (cmd *sliceNumpy) filterHGVScolpair(colpair [2][]int8) bool {
if cmd.chi2PValue >= 1 {
return true
if col < 4 && !cmd.includeVariant1 {
continue
}
+ if col&1 == 0 && cmd.pvalueMinFrequency < 1 && homhet2maf(obs[col:col+2]) < cmd.pvalueMinFrequency {
+ // Skip both columns (hom and het) if allele
+ // frequency is below threshold
+ col++
+ continue
+ }
atomic.AddInt64(&cmd.pvalueCallCount, 1)
p := cmd.pvalue(obs[col])
if cmd.chi2PValue < 1 && !(p < cmd.chi2PValue) {
return onehot, xref
}
+func homhet2maf(onehot [][]bool) float64 {
+ if len(onehot[0]) == 0 {
+ return 0
+ }
+ n := 0
+ for i := range onehot[0] {
+ if onehot[0][i] {
+ // hom
+ n += 2
+ } else if onehot[1][i] {
+ // het
+ n += 1
+ }
+ }
+ return float64(n) / float64(len(onehot[0])*2)
+}
+
// convert a []onehotXref with length N to a numpy-style []int32
// matrix with N columns, one row per field of onehotXref struct.
//
projects / lightning.git / blobdiff