6997: Avoid unnecessarily inefficient dirent sorting (and a mismatch between index...

[arvados.git] / services / keepstore / volume_unix.go

diff --git a/services/keepstore/volume_unix.go b/services/keepstore/volume_unix.go
index bcf57c1647ff54b2e5d2efd1432d54ab15be0559..a7ad6f9e499c80439c27cb1beed33060674ed776 100644 (file)
--- a/services/keepstore/volume_unix.go
+++ b/services/keepstore/volume_unix.go
@@ -9,6 +9,7 @@ import (
        "log"
        "os"
        "path/filepath"
+       "regexp"
        "strconv"
        "strings"
        "sync"
@@ -63,15 +64,33 @@ func (v *UnixVolume) Mtime(loc string) (time.Time, error) {
 // slice and whatever non-nil error was returned by Stat or ReadFile.
 func (v *UnixVolume) Get(loc string) ([]byte, error) {
        path := v.blockPath(loc)
-       if _, err := os.Stat(path); err != nil {
+       stat, err := os.Stat(path)
+       if err != nil {
+               return nil, err
+       }
+       if stat.Size() < 0 {
+               return nil, os.ErrInvalid
+       } else if stat.Size() == 0 {
+               return bufs.Get(0), nil
+       } else if stat.Size() > BLOCKSIZE {
+               return nil, TooLongError
+       }
+       f, err := os.Open(path)
+       if err != nil {
                return nil, err
        }
+       defer f.Close()
+       buf := bufs.Get(int(stat.Size()))
        if v.serialize {
                v.mutex.Lock()
                defer v.mutex.Unlock()
        }
-       buf, err := ioutil.ReadFile(path)
-       return buf, err
+       _, err = io.ReadFull(f, buf)
+       if err != nil {
+               bufs.Put(buf)
+               return nil, err
+       }
+       return buf, nil
 }
 
 // Put stores a block of data identified by the locator string
@@ -123,7 +142,7 @@ func (v *UnixVolume) Put(loc string, block []byte) error {
 }
 
 // Status returns a VolumeStatus struct describing the volume's
-// current state.
+// current state, or nil if an error occurs.
 //
 func (v *UnixVolume) Status() *VolumeStatus {
        var fs syscall.Statfs_t
@@ -149,6 +168,8 @@ func (v *UnixVolume) Status() *VolumeStatus {
        return &VolumeStatus{v.root, devnum, free, used}
 }
 
+var blockDirRe = regexp.MustCompile(`^[0-9a-f]+$`)
+
 // IndexTo writes (to the given Writer) a list of blocks found on this
 // volume which begin with the specified prefix. If the prefix is an
 // empty string, IndexTo writes a complete list of blocks.
@@ -164,31 +185,54 @@ func (v *UnixVolume) Status() *VolumeStatus {
 //     e4de7a2810f5554cd39b36d8ddb132ff+67108864 1388701136
 //
 func (v *UnixVolume) IndexTo(prefix string, w io.Writer) error {
-       return filepath.Walk(v.root,
-               func(path string, info os.FileInfo, err error) error {
-                       if err != nil {
-                               log.Printf("%s: IndexTo Walk error at %s: %s",
-                                       v, path, err)
-                               return nil
-                       }
-                       basename := filepath.Base(path)
-                       if info.IsDir() &&
-                               !strings.HasPrefix(basename, prefix) &&
-                               !strings.HasPrefix(prefix, basename) {
-                               // Skip directories that do not match
-                               // prefix. We know there is nothing
-                               // interesting inside.
-                               return filepath.SkipDir
+       var lastErr error = nil
+       rootdir, err := os.Open(v.root)
+       if err != nil {
+               return err
+       }
+       defer rootdir.Close()
+       for {
+               names, err := rootdir.Readdirnames(1)
+               if err == io.EOF {
+                       return lastErr
+               } else if err != nil {
+                       return err
+               }
+               if !strings.HasPrefix(names[0], prefix) && !strings.HasPrefix(prefix, names[0]) {
+                       // prefix excludes all blocks stored in this dir
+                       continue
+               }
+               if !blockDirRe.MatchString(names[0]) {
+                       continue
+               }
+               blockdirpath := filepath.Join(v.root, names[0])
+               blockdir, err := os.Open(blockdirpath)
+               if err != nil {
+                       log.Print("Error reading ", blockdirpath, ": ", err)
+                       lastErr = err
+                       continue
+               }
+               for {
+                       fileInfo, err := blockdir.Readdir(1)
+                       if err == io.EOF {
+                               break
+                       } else if err != nil {
+                               log.Print("Error reading ", blockdirpath, ": ", err)
+                               lastErr = err
+                               break
                        }
-                       if info.IsDir() ||
-                               !IsValidLocator(basename) ||
-                               !strings.HasPrefix(basename, prefix) {
-                               return nil
+                       name := fileInfo[0].Name()
+                       if !strings.HasPrefix(name, prefix) {
+                               continue
                        }
-                       _, err = fmt.Fprintf(w, "%s+%d %d\n",
-                               basename, info.Size(), info.ModTime().Unix())
-                       return err
-               })
+                       _, err = fmt.Fprint(w,
+                               name,
+                               "+", fileInfo[0].Size(),
+                               " ", fileInfo[0].ModTime().Unix(),
+                               "\n")
+               }
+               blockdir.Close()
+       }
 }
 
 func (v *UnixVolume) Delete(loc string) error {