Fix deadlock at container finish.

[lightning.git] / pca.go

diff --git a/pca.go b/pca.go
index f668e01557786e5d0829a14be5cb48d789bd44bb..121925d4c59d9833d37aabb397fa3a1bb0cc2243 100644 (file)
--- a/pca.go
+++ b/pca.go
@@ -1,7 +1,12 @@
-package main
+// Copyright (C) The Lightning Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package lightning
 
 import (
        "bufio"
+       "context"
        "errors"
        "flag"
        "fmt"
@@ -10,7 +15,7 @@ import (
        "net/http"
        _ "net/http/pprof"
        "os"
-       "sort"
+       "strings"
 
        "git.arvados.org/arvados.git/sdk/go/arvados"
        "github.com/james-bowman/nlp"
@@ -67,7 +72,9 @@ scipy.save(sys.argv[2], PCA(n_components=4).fit_transform(scipy.load(sys.argv[1]
        return 0
 }
 
-type goPCA struct{}
+type goPCA struct {
+       filter filter
+}
 
 func (cmd *goPCA) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
        var err error
@@ -86,6 +93,7 @@ func (cmd *goPCA) RunCommand(prog string, args []string, stdin io.Reader, stdout
        outputFilename := flags.String("o", "-", "output `file`")
        components := flags.Int("components", 4, "number of components")
        onehot := flags.Bool("one-hot", false, "recode tile variants as one-hot")
+       cmd.filter.Flags(flags)
        err = flags.Parse(args)
        if err == flag.ErrHelp {
                err = nil
@@ -109,8 +117,8 @@ func (cmd *goPCA) RunCommand(prog string, args []string, stdin io.Reader, stdout
                        Name:        "lightning pca-go",
                        Client:      arvados.NewClientFromEnv(),
                        ProjectUUID: *projectUUID,
-                       RAM:         432000000000,
-                       VCPUs:       2,
+                       RAM:         300000000000, // maybe 10x input size?
+                       VCPUs:       16,
                        Priority:    *priority,
                }
                err = runner.TranslatePaths(inputFilename)
@@ -118,6 +126,7 @@ func (cmd *goPCA) RunCommand(prog string, args []string, stdin io.Reader, stdout
                        return 1
                }
                runner.Args = []string{"pca-go", "-local=true", fmt.Sprintf("-one-hot=%v", *onehot), "-i", *inputFilename, "-o", "/mnt/output/pca.npy"}
+               runner.Args = append(runner.Args, cmd.filter.Args()...)
                var output string
                output, err = runner.Run()
                if err != nil {
@@ -138,7 +147,11 @@ func (cmd *goPCA) RunCommand(prog string, args []string, stdin io.Reader, stdout
                defer input.Close()
        }
        log.Print("reading")
-       cgs, err := ReadCompactGenomes(input)
+       tilelib := &tileLibrary{
+               retainNoCalls:  true,
+               compactGenomes: map[string][]tileVariantID{},
+       }
+       err = tilelib.LoadGob(context.Background(), input, strings.HasSuffix(*inputFilename, ".gz"))
        if err != nil {
                return 1
        }
@@ -146,16 +159,19 @@ func (cmd *goPCA) RunCommand(prog string, args []string, stdin io.Reader, stdout
        if err != nil {
                return 1
        }
-       log.Print("sorting")
-       sort.Slice(cgs, func(i, j int) bool { return cgs[i].Name < cgs[j].Name })
+
+       log.Info("filtering")
+       cmd.filter.Apply(tilelib)
+       log.Info("tidying")
+       tilelib.Tidy()
 
        log.Print("converting cgs to array")
-       data, rows, cols := cgs2array(cgs)
+       data, rows, cols := cgs2array(tilelib, cgnames(tilelib), lowqual(tilelib), nil, 0, len(tilelib.variant))
        if *onehot {
                log.Printf("recode one-hot: %d rows, %d cols", rows, cols)
-               data, cols = recodeOnehot(data, cols)
+               data, _, cols = recodeOnehot(data, cols)
        }
-       cgs = nil
+       tilelib = nil
 
        log.Printf("creating matrix backed by array: %d rows, %d cols", rows, cols)
        mtx := array2matrix(rows, cols, data).T()
@@ -209,7 +225,7 @@ func (cmd *goPCA) RunCommand(prog string, args []string, stdin io.Reader, stdout
        return 0
 }
 
-func array2matrix(rows, cols int, data []uint16) mat.Matrix {
+func array2matrix(rows, cols int, data []int16) mat.Matrix {
        floatdata := make([]float64, rows*cols)
        for i, v := range data {
                floatdata[i] = float64(v)