Fixup multiple-file reading.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/arvados.go b/arvados.go
index 0eb99686aa748969f4000c47ede9681869971e05..ec2701bcdc7c96664601351225aedb83f3b5e16a 100644 (file)
--- a/arvados.go
+++ b/arvados.go
@@ -528,8 +528,8 @@ func open(fnm string) (file, error) {
                return os.Open(fnm)
        }
        uuid := m[2]
-       mnt := "/mnt/" + uuid + "/"
-       if !strings.HasPrefix(fnm, mnt) {
+       mnt := "/mnt/" + uuid
+       if fnm != mnt && !strings.HasPrefix(fnm, mnt+"/") {
                return os.Open(fnm)
        }
 
@@ -552,5 +552,5 @@ func open(fnm string) (file, error) {
        }
 
        log.Infof("reading %q from %s using Arvados client", fnm[len(mnt):], uuid)
-       return siteFS.Open("by_id/" + uuid + "/" + fnm[len(mnt):])
+       return siteFS.Open("by_id/" + uuid + fnm[len(mnt):])
 }

diff --git a/gob.go b/gob.go
index 88b78c78c973632aebac2980ae6f568d7f39c8cd..acc426de96c7c2af7f50b4a6ee133d9cfdbf0d95 100644 (file)
--- a/gob.go
+++ b/gob.go
@@ -48,7 +48,7 @@ func DecodeLibrary(rdr io.Reader, gz bool, cb func(*LibraryEntry) error) error {
        zrdr := ioutil.NopCloser(rdr)
        var err error
        if gz {
-               zrdr, err = pgzip.NewReader(bufio.NewReaderSize(rdr, 1<<26))
+               zrdr, err = pgzip.NewReader(bufio.NewReaderSize(rdr, 1<<20))
                if err != nil {
                        return err
                }

diff --git a/tilelib.go b/tilelib.go
index baeb259ea084c25014c57cbd0202a91f2e8c7321..b3c43f181122e42a55eacf8a0dcca1bfb4a0e676 100644 (file)
--- a/tilelib.go
+++ b/tilelib.go
@@ -227,17 +227,18 @@ func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGeno
                for _, fi := range fis {
                        if fi.Name() == "." || fi.Name() == ".." {
                                continue
-                       } else if fi.IsDir() {
-                               err = walk(path + "/" + fi.Name())
+                       } else if child := path + "/" + fi.Name(); fi.IsDir() {
+                               err = walk(child)
                                if err != nil {
                                        return err
                                }
-                       } else if strings.HasSuffix(path, ".gob") || strings.HasSuffix(path, ".gob.gz") {
-                               files = append(files, path)
+                       } else if strings.HasSuffix(child, ".gob") || strings.HasSuffix(child, ".gob.gz") {
+                               files = append(files, child)
                        }
                }
                return nil
        }
+       log.Infof("LoadDir: walk dir %s", path)
        err := walk(path)
        if err != nil {
                return err
@@ -249,6 +250,7 @@ func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGeno
        cseqs := []CompactSequence{}
        variantmap := map[tileLibRef]tileVariantID{}
        errs := make(chan error, len(files))
+       log.Infof("LoadDir: read %d files", len(files))
        for _, path := range files {
                path := path
                go func() {
@@ -264,8 +266,13 @@ func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGeno
                                }
                                mtx.Lock()
                                defer mtx.Unlock()
-                               if err := tilelib.loadTagSet(ent.TagSet); err != nil {
-                                       return err
+                               if tilelib.taglib == nil || tilelib.taglib.Len() != len(ent.TagSet) {
+                                       // load first set of tags, or
+                                       // report mismatch if 2 sets
+                                       // have different #tags.
+                                       if err := tilelib.loadTagSet(ent.TagSet); err != nil {
+                                               return err
+                                       }
                                }
                                if err := tilelib.loadTileVariants(ent.TileVariants, variantmap); err != nil {
                                        return err
@@ -282,14 +289,17 @@ func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string, onLoadGeno
                        return err
                }
        }
+       log.Info("LoadDir: loadCompactGenomes")
        err = tilelib.loadCompactGenomes(cgs, variantmap, onLoadGenome)
        if err != nil {
                return err
        }
+       log.Info("LoadDir: loadCompactSequences")
        err = tilelib.loadCompactSequences(cseqs, variantmap)
        if err != nil {
                return err
        }
+       log.Info("LoadDir done")
        return nil
 }