Client *arvados.Client
Name string
ProjectUUID string
+ VCPUs int
+ RAM int64
Args []string
Mounts map[string]string
}
"uuid": uuid,
}
}
- cpus := 16
rc := arvados.RuntimeConstraints{
- VCPUs: cpus,
- RAM: 64000000000,
- KeepCacheRAM: (1 << 26) * 2 * int64(cpus),
+ VCPUs: runner.VCPUs,
+ RAM: runner.RAM,
+ KeepCacheRAM: (1 << 26) * 2 * int64(runner.VCPUs),
}
var cr arvados.ContainerRequest
err = runner.Client.RequestAndDecode(&cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{
Limit: 1,
Count: "none",
Filters: []arvados.Filter{
- {"name", "=", cname},
- {"owner_uuid", "=", runner.ProjectUUID},
+ {Attr: "name", Operator: "=", Operand: cname},
+ {Attr: "owner_uuid", Operator: "=", Operand: runner.ProjectUUID},
},
})
if err != nil {
import (
"bufio"
+ "errors"
"flag"
"fmt"
"io"
"log"
"net/http"
_ "net/http/pprof"
+ "os"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/kshedden/gonpy"
)
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
pprof := flags.String("pprof", "", "serve Go profile data at http://`[addr]:port`")
+ runlocal := flags.Bool("local", false, "run on local host (default: run in an arvados container)")
+ projectUUID := flags.String("project", "", "project `UUID` for output data")
+ inputFilename := flags.String("i", "", "input `file`")
+ outputFilename := flags.String("o", "", "output `file`")
err = flags.Parse(args)
if err == flag.ErrHelp {
err = nil
}()
}
+ if !*runlocal {
+ if *outputFilename != "" {
+ err = errors.New("cannot specify output file in container mode: not implemented")
+ return 1
+ }
+ runner := arvadosContainerRunner{
+ Name: "lightning export-numpy",
+ Client: arvados.NewClientFromEnv(),
+ ProjectUUID: *projectUUID,
+ RAM: 64000000000,
+ VCPUs: 2,
+ }
+ err = runner.TranslatePaths(inputFilename)
+ if err != nil {
+ return 1
+ }
+ runner.Args = []string{"export-numpy", "-local=true", "-i", *inputFilename, "-o", "/mnt/output/library.npy"}
+ err = runner.Run()
+ if err != nil {
+ return 1
+ }
+ return 0
+ }
+
cgs, err := ReadCompactGenomes(stdin)
if err != nil {
return 1
out[row*cols+i] = uint16(v)
}
}
- w := bufio.NewWriter(cmd.output)
- npw, err := gonpy.NewWriter(nopCloser{w})
+
+ var output io.WriteCloser
+ if *outputFilename == "" {
+ output = nopCloser{cmd.output}
+ } else {
+ output, err = os.OpenFile(*outputFilename, os.O_CREATE|os.O_WRONLY, 0777)
+ if err != nil {
+ return 1
+ }
+ defer output.Close()
+ }
+ bufw := bufio.NewWriter(output)
+ npw, err := gonpy.NewWriter(nopCloser{bufw})
if err != nil {
return 1
}
npw.Shape = []int{rows, cols}
npw.WriteUint16(out)
- err = w.Flush()
+ err = bufw.Flush()
+ if err != nil {
+ return 1
+ }
+ err = output.Close()
if err != nil {
return 1
}
exited := (&importer{}).RunCommand("import", []string{"-local=true", "-tag-library", "testdata/tags", "-ref", "testdata/ref", "testdata/a.1.fasta"}, &bytes.Buffer{}, &buffer, os.Stderr)
c.Assert(exited, check.Equals, 0)
var output bytes.Buffer
- exited = (&exportNumpy{}).RunCommand("export-numpy", nil, &buffer, &output, os.Stderr)
+ exited = (&exportNumpy{}).RunCommand("export-numpy", []string{"-local=true"}, &buffer, &output, os.Stderr)
c.Check(exited, check.Equals, 0)
npy, err := gonpy.NewReader(&output)
c.Assert(err, check.IsNil)
import (
"bufio"
"encoding/gob"
+ "errors"
"flag"
"fmt"
"io"
+ "io/ioutil"
"log"
"net/http"
_ "net/http/pprof"
+ "os"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
)
type filterer struct {
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
pprof := flags.String("pprof", "", "serve Go profile data at http://`[addr]:port`")
+ runlocal := flags.Bool("local", false, "run on local host (default: run in an arvados container)")
+ projectUUID := flags.String("project", "", "project `UUID` for output data")
+ inputFilename := flags.String("i", "", "input `file`")
+ outputFilename := flags.String("o", "", "output `file`")
maxvariants := flags.Int("max-variants", -1, "drop tiles with more than `N` variants")
mincoverage := flags.Float64("min-coverage", 1, "drop tiles with coverage less than `P` across all haplotypes (0 < P ≤ 1)")
maxtag := flags.Int("max-tag", -1, "drop tiles with tag ID > `N`")
}()
}
+ if !*runlocal {
+ if *outputFilename != "" {
+ err = errors.New("cannot specify output file in container mode: not implemented")
+ return 1
+ }
+ runner := arvadosContainerRunner{
+ Name: "lightning filter",
+ Client: arvados.NewClientFromEnv(),
+ ProjectUUID: *projectUUID,
+ RAM: 64000000000,
+ VCPUs: 2,
+ }
+ err = runner.TranslatePaths(inputFilename)
+ if err != nil {
+ return 1
+ }
+ runner.Args = []string{"filter", "-local=true",
+ "-i", *inputFilename,
+ "-o", "/mnt/output/library.gob",
+ "-max-variants", fmt.Sprintf("%d", *maxvariants),
+ "-min-coverage", fmt.Sprintf("%f", *mincoverage),
+ "-max-tag", fmt.Sprintf("%d", *maxtag),
+ }
+ err = runner.Run()
+ if err != nil {
+ return 1
+ }
+ return 0
+ }
+
+ var infile io.ReadCloser
+ if *inputFilename == "" {
+ infile = ioutil.NopCloser(stdin)
+ } else {
+ infile, err = os.Open(*inputFilename)
+ if err != nil {
+ return 1
+ }
+ defer infile.Close()
+ }
log.Print("reading")
- cgs, err := ReadCompactGenomes(stdin)
+ cgs, err := ReadCompactGenomes(infile)
+ if err != nil {
+ return 1
+ }
+ err = infile.Close()
if err != nil {
return 1
}
log.Print("filtering done")
- w := bufio.NewWriter(cmd.output)
+ var outfile io.WriteCloser
+ if *outputFilename == "" {
+ outfile = nopCloser{cmd.output}
+ } else {
+ outfile, err = os.OpenFile(*outputFilename, os.O_CREATE|os.O_WRONLY, 0777)
+ if err != nil {
+ return 1
+ }
+ defer outfile.Close()
+ }
+ w := bufio.NewWriter(outfile)
enc := gob.NewEncoder(w)
log.Print("writing")
err = enc.Encode(LibraryEntry{
if err != nil {
return 1
}
+ err = outfile.Close()
+ if err != nil {
+ return 1
+ }
return 0
}
flags.StringVar(&cmd.tagLibraryFile, "tag-library", "", "tag library fasta `file`")
flags.StringVar(&cmd.refFile, "ref", "", "reference fasta `file`")
flags.StringVar(&cmd.outputFile, "o", "", "output `file`")
- flags.StringVar(&cmd.projectUUID, "project", "", "project `UUID` for storing intermediate and output data")
+ flags.StringVar(&cmd.projectUUID, "project", "", "project `UUID` for output data")
flags.BoolVar(&cmd.runLocal, "local", false, "run on local host (default: run in an arvados container)")
pprof := flags.String("pprof", "", "serve Go profile data at http://`[addr]:port`")
err = flags.Parse(args)
Name: "lightning import",
Client: arvados.NewClientFromEnv(),
ProjectUUID: cmd.projectUUID,
+ RAM: 30000000000,
+ VCPUs: 16,
}
err = runner.TranslatePaths(&cmd.tagLibraryFile, &cmd.refFile, &cmd.outputFile)
if err != nil {
}
}()
- var outfile *os.File
- var w *bufio.Writer
+ var output io.WriteCloser
if cmd.outputFile == "" {
- w = bufio.NewWriter(stdout)
+ output = nopCloser{stdout}
} else {
- outfile, err = os.OpenFile(cmd.outputFile, os.O_CREATE|os.O_WRONLY, 0777)
+ output, err = os.OpenFile(cmd.outputFile, os.O_CREATE|os.O_WRONLY, 0777)
if err != nil {
return 1
}
- defer outfile.Close()
- w = bufio.NewWriter(outfile)
+ defer output.Close()
}
- cmd.encoder = gob.NewEncoder(w)
+ bufw := bufio.NewWriter(output)
+ cmd.encoder = gob.NewEncoder(bufw)
err = cmd.tileInputs(tilelib, infiles)
if err != nil {
return 1
}
- err = w.Flush()
+ err = bufw.Flush()
if err != nil {
return 1
}
- if outfile != nil {
- err = outfile.Close()
- if err != nil {
- return 1
- }
+ err = output.Close()
+ if err != nil {
+ return 1
}
return 0
}