flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.SetOutput(stderr)
pprof := flags.String("pprof", "", "serve Go profile data at http://`[addr]:port`")
+ pprofdir := flags.String("pprof-dir", "", "write Go profile data to `directory` periodically")
runlocal := flags.Bool("local", false, "run on local host (default: run in an arvados container)")
projectUUID := flags.String("project", "", "project `UUID` for output data")
priority := flags.Int("priority", 500, "container request priority")
+ preemptible := flags.Bool("preemptible", true, "request preemptible instance")
outputDir := flags.String("output-dir", "./out", "output `directory`")
tagsPerFile := flags.Int("tags-per-file", 50000, "tags per file (nfiles will be ~10M÷x)")
err = flags.Parse(args)
log.Println(http.ListenAndServe(*pprof, nil))
}()
}
+ if *pprofdir != "" {
+ go writeProfilesPeriodically(*pprofdir)
+ }
if !*runlocal {
runner := arvadosContainerRunner{
Priority: *priority,
KeepCache: 2,
APIAccess: true,
+ Preemptible: *preemptible,
}
for i := range inputDirs {
err = runner.TranslatePaths(&inputDirs[i])
func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
var infiles []string
for _, srcdir := range srcdirs {
- files, err := allGobFiles(srcdir)
+ files, err := allFiles(srcdir, matchGobFile)
if err != nil {
return err
}
throttle := throttle{Max: runtime.GOMAXPROCS(0)}
for _, infile := range infiles {
infile := infile
- throttle.Acquire()
- go func() {
- defer throttle.Release()
+ throttle.Go(func() error {
f, err := open(infile)
if err != nil {
- throttle.Report(err)
- return
+ return err
}
defer f.Close()
dir, _ := filepath.Split(infile)
namespace := dirNamespace[dir]
log.Printf("reading %s (namespace %d)", infile, namespace)
- err = DecodeLibrary(f, strings.HasSuffix(infile, ".gz"), func(ent *LibraryEntry) error {
+ return DecodeLibrary(f, strings.HasSuffix(infile, ".gz"), func(ent *LibraryEntry) error {
if err := throttle.Err(); err != nil {
return err
}
}
return nil
})
- throttle.Report(err)
- }()
+ })
}
throttle.Wait()
if throttle.Err() != nil {