Container request priority flag.

[lightning.git] / import.go

diff --git a/import.go b/import.go
index 14b3667ed7fec34df0c75a4ef5504b4dc356d838..e6b86afd81ca22038c025c669e39a6fd3a0c3b88 100644 (file)
--- a/import.go
+++ b/import.go
@@ -8,7 +8,6 @@ import (
        "flag"
        "fmt"
        "io"
-       "log"
        "net/http"
        _ "net/http/pprof"
        "os"
@@ -23,6 +22,7 @@ import (
        "time"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
+       log "github.com/sirupsen/logrus"
 )
 
 type importer struct {
@@ -45,9 +45,10 @@ func (cmd *importer) RunCommand(prog string, args []string, stdin io.Reader, std
        flags.SetOutput(stderr)
        flags.StringVar(&cmd.tagLibraryFile, "tag-library", "", "tag library fasta `file`")
        flags.StringVar(&cmd.refFile, "ref", "", "reference fasta `file`")
-       flags.StringVar(&cmd.outputFile, "o", "", "output `file`")
+       flags.StringVar(&cmd.outputFile, "o", "-", "output `file`")
        flags.StringVar(&cmd.projectUUID, "project", "", "project `UUID` for output data")
        flags.BoolVar(&cmd.runLocal, "local", false, "run on local host (default: run in an arvados container)")
+       priority := flags.Int("priority", 500, "container request priority")
        pprof := flags.String("pprof", "", "serve Go profile data at http://`[addr]:port`")
        err = flags.Parse(args)
        if err == flag.ErrHelp {
@@ -76,6 +77,7 @@ func (cmd *importer) RunCommand(prog string, args []string, stdin io.Reader, std
                        ProjectUUID: cmd.projectUUID,
                        RAM:         30000000000,
                        VCPUs:       16,
+                       Priority:    *priority,
                }
                err = runner.TranslatePaths(&cmd.tagLibraryFile, &cmd.refFile, &cmd.outputFile)
                if err != nil {
@@ -88,7 +90,7 @@ func (cmd *importer) RunCommand(prog string, args []string, stdin io.Reader, std
                                return 1
                        }
                }
-               if cmd.outputFile == "" {
+               if cmd.outputFile == "-" {
                        cmd.outputFile = "/mnt/output/library.gob"
                } else {
                        // Not yet implemented, but this should write
@@ -98,10 +100,12 @@ func (cmd *importer) RunCommand(prog string, args []string, stdin io.Reader, std
                        return 1
                }
                runner.Args = append([]string{"import", "-local=true", "-tag-library", cmd.tagLibraryFile, "-ref", cmd.refFile, "-o", cmd.outputFile}, inputs...)
-               err = runner.Run()
+               var output string
+               output, err = runner.Run()
                if err != nil {
                        return 1
                }
+               fmt.Fprintln(stdout, output+"/library.gob")
                return 0
        }
 
@@ -115,13 +119,13 @@ func (cmd *importer) RunCommand(prog string, args []string, stdin io.Reader, std
                return 1
        }
        go func() {
-               for range time.Tick(10 * time.Second) {
+               for range time.Tick(10 * time.Minute) {
                        log.Printf("tilelib.Len() == %d", tilelib.Len())
                }
        }()
 
        var output io.WriteCloser
-       if cmd.outputFile == "" {
+       if cmd.outputFile == "-" {
                output = nopCloser{stdout}
        } else {
                output, err = os.OpenFile(cmd.outputFile, os.O_CREATE|os.O_WRONLY, 0777)
@@ -305,9 +309,10 @@ func (cmd *importer) tileInputs(tilelib *tileLibrary, infiles []string) error {
        }
        go close(todo)
        var tileJobs sync.WaitGroup
-       running := int64(runtime.NumCPU())
-       for i := 0; i < runtime.NumCPU(); i++ {
+       var running int64
+       for i := 0; i < runtime.NumCPU()*9/8+1; i++ {
                tileJobs.Add(1)
+               atomic.AddInt64(&running, 1)
                go func() {
                        defer tileJobs.Done()
                        defer atomic.AddInt64(&running, -1)