Bump import memory.

[lightning.git] / exportnumpy.go

diff --git a/exportnumpy.go b/exportnumpy.go
index eba0234bfe216e12384306f73b97fec04e34a673..99b0381c93c615128aa1d2d2508c9d1f1e4ccc5d 100644 (file)
--- a/exportnumpy.go
+++ b/exportnumpy.go
@@ -11,6 +11,7 @@ import (
        "net/http"
        _ "net/http/pprof"
        "os"
+       "path"
        "sort"
        "strings"
 
@@ -66,9 +67,11 @@ func (cmd *exportNumpy) RunCommand(prog string, args []string, stdin io.Reader,
                        Name:        "lightning export-numpy",
                        Client:      arvados.NewClientFromEnv(),
                        ProjectUUID: *projectUUID,
-                       RAM:         128000000000,
+                       RAM:         450000000000,
                        VCPUs:       32,
                        Priority:    *priority,
+                       KeepCache:   1,
+                       APIAccess:   true,
                }
                err = runner.TranslatePaths(inputFilename)
                if err != nil {
@@ -98,12 +101,13 @@ func (cmd *exportNumpy) RunCommand(prog string, args []string, stdin io.Reader,
        if *inputFilename == "-" {
                input = ioutil.NopCloser(stdin)
        } else {
-               input, err = os.Open(*inputFilename)
+               input, err = open(*inputFilename)
                if err != nil {
                        return 1
                }
                defer input.Close()
        }
+       input = ioutil.NopCloser(bufio.NewReaderSize(input, 8*1024*1024))
        tilelib := &tileLibrary{
                retainNoCalls:       true,
                retainTileSequences: true,
@@ -152,8 +156,9 @@ func (cmd *exportNumpy) RunCommand(prog string, args []string, stdin io.Reader,
                        return 1
                }
                defer f.Close()
+               _, outBasename := path.Split(*outputFilename)
                for i, name := range names {
-                       _, err = fmt.Fprintf(f, "%d,%q\n", i, trimFilenameForLabel(name))
+                       _, err = fmt.Fprintf(f, "%d,%q,%q\n", i, trimFilenameForLabel(name), outBasename)
                        if err != nil {
                                err = fmt.Errorf("write %s: %w", *labelsFilename, err)
                                return 1
@@ -227,7 +232,9 @@ func cgs2array(tilelib *tileLibrary) (data []int16, rows, cols int, cgnames []st
        for name := range tilelib.compactGenomes {
                cgnames = append(cgnames, name)
        }
-       sort.Strings(cgnames)
+       sort.Slice(cgnames, func(i, j int) bool {
+               return trimFilenameForLabel(cgnames[i]) < trimFilenameForLabel(cgnames[j])
+       })
 
        rows = len(tilelib.compactGenomes)
        for _, cg := range tilelib.compactGenomes {