Limit load goroutines to gomaxprocs/2.

Otherwise we bottleneck early on lots of IO that we can't process that
quickly.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/tilelib.go b/tilelib.go
index 3852b4d093d6175294477c724f764d65dba5b3e8..8960d0e86d8659bf94335ab0bb86b8b5c4aced11 100644 (file)
--- a/tilelib.go
+++ b/tilelib.go
@@ -251,9 +251,12 @@ func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGeno
        variantmap := map[tileLibRef]tileVariantID{}
        errs := make(chan error, len(files))
        log.Infof("LoadDir: read %d files", len(files))
+       throttle := throttle{Max: runtime.GOMAXPROCS(0)/2 + 1}
        for _, path := range files {
                path := path
                go func() {
+                       throttle.Acquire()
+                       defer throttle.Release()
                        f, err := open(path)
                        if err != nil {
                                errs <- err