Not a bug when ref is the only variant loaded.

[lightning.git] / slice.go

diff --git a/slice.go b/slice.go
index be5bb87904f057ea0a0310358f9371e80d081ccb..d2073ad9c05f8900bc55dd8780e30bb6ca8cc699 100644 (file)
--- a/slice.go
+++ b/slice.go
@@ -66,10 +66,10 @@ func (cmd *slicecmd) RunCommand(prog string, args []string, stdin io.Reader, std
                        Name:        "lightning slice",
                        Client:      arvados.NewClientFromEnv(),
                        ProjectUUID: *projectUUID,
-                       RAM:         200000000000,
-                       VCPUs:       32,
+                       RAM:         500000000000,
+                       VCPUs:       64,
                        Priority:    *priority,
-                       KeepCache:   50,
+                       KeepCache:   2,
                        APIAccess:   true,
                }
                for i := range inputDirs {
@@ -111,14 +111,14 @@ func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
        }
        // dirNamespace[dir] is an int in [0,len(dirNamespace)), used below to
        // namespace variant numbers from different dirs.
-       dirNamespace := map[string]int{}
+       dirNamespace := map[string]tileVariantID{}
        for _, path := range infiles {
                dir, _ := filepath.Split(path)
                if _, ok := dirNamespace[dir]; !ok {
-                       dirNamespace[dir] = len(dirNamespace)
+                       dirNamespace[dir] = tileVariantID(len(dirNamespace))
                }
        }
-       namespaces := len(dirNamespace)
+       namespaces := tileVariantID(len(dirNamespace))
 
        var (
                tagset     [][]byte
@@ -134,21 +134,21 @@ func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
        )
 
        throttle := throttle{Max: runtime.GOMAXPROCS(0)}
-       for _, path := range infiles {
-               path := path
+       for _, infile := range infiles {
+               infile := infile
                throttle.Acquire()
                go func() {
                        defer throttle.Release()
-                       f, err := open(path)
+                       f, err := open(infile)
                        if err != nil {
                                throttle.Report(err)
                                return
                        }
                        defer f.Close()
-                       dir, _ := filepath.Split(path)
+                       dir, _ := filepath.Split(infile)
                        namespace := dirNamespace[dir]
-                       log.Printf("reading %s (namespace %d)", path, namespace)
-                       err = DecodeLibrary(f, strings.HasSuffix(path, ".gz"), func(ent *LibraryEntry) error {
+                       log.Printf("reading %s (namespace %d)", infile, namespace)
+                       err = DecodeLibrary(f, strings.HasSuffix(infile, ".gz"), func(ent *LibraryEntry) error {
                                if err := throttle.Err(); err != nil {
                                        return err
                                }
@@ -175,8 +175,12 @@ func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
                                }
                                atomic.AddInt64(&countTileVariants, int64(len(ent.TileVariants)))
                                for _, tv := range ent.TileVariants {
-                                       tv.Variant = tileVariantID(int(tv.Variant)*namespaces + namespace)
-                                       err := encs[int(tv.Tag)/tagsPerFile].Encode(LibraryEntry{
+                                       tv.Variant = tv.Variant*namespaces + namespace
+                                       fileno := 0
+                                       if !tv.Ref {
+                                               fileno = int(tv.Tag) / tagsPerFile
+                                       }
+                                       err := encs[fileno].Encode(LibraryEntry{
                                                TileVariants: []TileVariant{tv},
                                        })
                                        if err != nil {
@@ -192,7 +196,7 @@ func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
                                for _, cg := range ent.CompactGenomes {
                                        for i, v := range cg.Variants {
                                                if v > 0 {
-                                                       cg.Variants[i] = tileVariantID(int(v)*namespaces + namespace)
+                                                       cg.Variants[i] = v*namespaces + namespace
                                                }
                                        }
                                        for i, enc := range encs {
@@ -226,7 +230,7 @@ func Slice(tagsPerFile int, dstdir string, srcdirs []string) error {
                                        for _, cs := range ent.CompactSequences {
                                                for _, tseq := range cs.TileSequences {
                                                        for i, libref := range tseq {
-                                                               tseq[i].Variant = tileVariantID(int(libref.Variant)*namespaces + namespace)
+                                                               tseq[i].Variant = libref.Variant*namespaces + namespace
                                                        }
                                                }
                                        }