Fix some tests.

[lightning.git] / slice.go

diff --git a/slice.go b/slice.go
index fed3766dfe0437285fbeb20e150291ba073c729c..63e751c8da9aa9a245a64d94c3948c4e0443c06e 100644 (file)
--- a/slice.go
+++ b/slice.go
@@ -37,9 +37,11 @@ func (cmd *slicecmd) RunCommand(prog string, args []string, stdin io.Reader, std
        flags := flag.NewFlagSet("", flag.ContinueOnError)
        flags.SetOutput(stderr)
        pprof := flags.String("pprof", "", "serve Go profile data at http://`[addr]:port`")
+       pprofdir := flags.String("pprof-dir", "", "write Go profile data to `directory` periodically")
        runlocal := flags.Bool("local", false, "run on local host (default: run in an arvados container)")
        projectUUID := flags.String("project", "", "project `UUID` for output data")
        priority := flags.Int("priority", 500, "container request priority")
+       preemptible := flags.Bool("preemptible", true, "request preemptible instance")
        outputDir := flags.String("output-dir", "./out", "output `directory`")
        tagsPerFile := flags.Int("tags-per-file", 50000, "tags per file (nfiles will be ~10M÷x)")
        err = flags.Parse(args)
@@ -60,17 +62,21 @@ func (cmd *slicecmd) RunCommand(prog string, args []string, stdin io.Reader, std
                        log.Println(http.ListenAndServe(*pprof, nil))
                }()
        }
+       if *pprofdir != "" {
+               go writeProfilesPeriodically(*pprofdir)
+       }
 
        if !*runlocal {
                runner := arvadosContainerRunner{
                        Name:        "lightning slice",
                        Client:      arvados.NewClientFromEnv(),
                        ProjectUUID: *projectUUID,
-                       RAM:         300000000000,
-                       VCPUs:       32,
+                       RAM:         500000000000,
+                       VCPUs:       64,
                        Priority:    *priority,
                        KeepCache:   2,
                        APIAccess:   true,
+                       Preemptible: *preemptible,
                }
                for i := range inputDirs {
                        err = runner.TranslatePaths(&inputDirs[i])
@@ -103,7 +109,7 @@ func (cmd *slicecmd) RunCommand(prog string, args []string, stdin io.Reader, std
 func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
        var infiles []string
        for _, srcdir := range srcdirs {
-               files, err := allGobFiles(srcdir)
+               files, err := allFiles(srcdir, matchGobFile)
                if err != nil {
                        return err
                }
@@ -136,19 +142,16 @@ func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
        throttle := throttle{Max: runtime.GOMAXPROCS(0)}
        for _, infile := range infiles {
                infile := infile
-               throttle.Acquire()
-               go func() {
-                       defer throttle.Release()
+               throttle.Go(func() error {
                        f, err := open(infile)
                        if err != nil {
-                               throttle.Report(err)
-                               return
+                               return err
                        }
                        defer f.Close()
                        dir, _ := filepath.Split(infile)
                        namespace := dirNamespace[dir]
                        log.Printf("reading %s (namespace %d)", infile, namespace)
-                       err = DecodeLibrary(f, strings.HasSuffix(infile, ".gz"), func(ent *LibraryEntry) error {
+                       return DecodeLibrary(f, strings.HasSuffix(infile, ".gz"), func(ent *LibraryEntry) error {
                                if err := throttle.Err(); err != nil {
                                        return err
                                }
@@ -241,8 +244,7 @@ func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
                                }
                                return nil
                        })
-                       throttle.Report(err)
-               }()
+               })
        }
        throttle.Wait()
        if throttle.Err() != nil {