12216: Avoid opening all files when generating webdav dir lists.

[arvados.git] / services / keep-web / webdav.go

diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 0a7b7822b20c7f8f5be5cea10eff434481cfe3f8..57f3f53a99ef6a73944c6cb600c1672c67cb6696 100644 (file)
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -11,9 +11,12 @@ import (
        prand "math/rand"
        "net/http"
        "os"
+       "sync"
        "sync/atomic"
        "time"
 
+       "git.curoverse.com/arvados.git/sdk/go/arvados"
+
        "golang.org/x/net/context"
        "golang.org/x/net/webdav"
 )
@@ -24,10 +27,10 @@ var (
        errReadOnly           = errors.New("read-only filesystem")
 )
 
-// webdavFS implements a read-only webdav.FileSystem by wrapping
-// http.Filesystem.
+// webdavFS implements a read-only webdav.FileSystem by wrapping an
+// arvados.CollectionFilesystem.
 type webdavFS struct {
-       httpfs http.FileSystem
+       collfs arvados.CollectionFileSystem
 }
 
 var _ webdav.FileSystem = &webdavFS{}
@@ -37,11 +40,11 @@ func (fs *webdavFS) Mkdir(ctx context.Context, name string, perm os.FileMode) er
 }
 
 func (fs *webdavFS) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
-       f, err := fs.httpfs.Open(name)
+       fi, err := fs.collfs.Stat(name)
        if err != nil {
                return nil, err
        }
-       return &webdavFile{File: f}, nil
+       return &webdavFile{collfs: fs.collfs, fileInfo: fi, name: name}, nil
 }
 
 func (fs *webdavFS) RemoveAll(ctx context.Context, name string) error {
@@ -53,23 +56,76 @@ func (fs *webdavFS) Rename(ctx context.Context, oldName, newName string) error {
 }
 
 func (fs *webdavFS) Stat(ctx context.Context, name string) (os.FileInfo, error) {
-       if f, err := fs.httpfs.Open(name); err != nil {
-               return nil, err
-       } else {
-               return f.Stat()
-       }
+       return fs.collfs.Stat(name)
 }
 
 // webdavFile implements a read-only webdav.File by wrapping
-// http.File. Writes fail.
+// http.File.
+//
+// The http.File is opened from an arvados.CollectionFileSystem, but
+// not until Seek, Read, or Readdir is called. This deferred-open
+// strategy makes webdav's OpenFile-Stat-Close cycle fast even though
+// the collfs's Open method is slow. This is relevant because webdav
+// does OpenFile-Stat-Close on each file when preparing directory
+// listings.
+//
+// Writes to a webdavFile always fail.
 type webdavFile struct {
-       http.File
+       // fields populated by (*webdavFS).OpenFile()
+       collfs   http.FileSystem
+       fileInfo os.FileInfo
+       name     string
+
+       // internal fields
+       file     http.File
+       loadOnce sync.Once
+       err      error
+}
+
+func (f *webdavFile) load() {
+       f.file, f.err = f.collfs.Open(f.name)
 }
 
 func (f *webdavFile) Write([]byte) (int, error) {
        return 0, errReadOnly
 }
 
+func (f *webdavFile) Seek(offset int64, whence int) (int64, error) {
+       f.loadOnce.Do(f.load)
+       if f.err != nil {
+               return 0, f.err
+       }
+       return f.file.Seek(offset, whence)
+}
+
+func (f *webdavFile) Read(buf []byte) (int, error) {
+       f.loadOnce.Do(f.load)
+       if f.err != nil {
+               return 0, f.err
+       }
+       return f.file.Read(buf)
+}
+
+func (f *webdavFile) Close() error {
+       if f.file == nil {
+               // We never called load(), or load() failed
+               return f.err
+       }
+       return f.file.Close()
+}
+
+func (f *webdavFile) Readdir(n int) ([]os.FileInfo, error) {
+       f.loadOnce.Do(f.load)
+       if f.err != nil {
+               return nil, f.err
+       }
+       return f.file.Readdir(n)
+}
+
+func (f *webdavFile) Stat() (os.FileInfo, error) {
+       return f.fileInfo, nil
+}
+
 // noLockSystem implements webdav.LockSystem by returning success for
 // every possible locking operation, even though it has no side
 // effects such as actually locking anything. This works for a