package arvados
import (
- "crypto/md5"
"errors"
"fmt"
"io"
)
var (
- ErrReadOnlyFile = errors.New("read-only file")
- ErrNegativeOffset = errors.New("cannot seek to negative offset")
- ErrFileExists = errors.New("file exists")
- ErrInvalidOperation = errors.New("invalid operation")
- ErrPermission = os.ErrPermission
+ ErrReadOnlyFile = errors.New("read-only file")
+ ErrNegativeOffset = errors.New("cannot seek to negative offset")
+ ErrFileExists = errors.New("file exists")
+ ErrInvalidOperation = errors.New("invalid operation")
+ ErrInvalidArgument = errors.New("invalid argument")
+ ErrDirectoryNotEmpty = errors.New("directory not empty")
+ ErrWriteOnlyMode = errors.New("file is O_WRONLY")
+ ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
+ ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
+ ErrPermission = os.ErrPermission
maxBlockSize = 1 << 26
)
type keepClient interface {
ReadAt(locator string, p []byte, off int) (int, error)
+ PutB(p []byte) (string, int, error)
}
type fileinfo struct {
return nil
}
-func (fi fileinfo) Stat() os.FileInfo {
- return fi
-}
-
// A CollectionFileSystem is an http.Filesystem plus Stat() and
-// support for opening writable files.
+// support for opening writable files. All methods are safe to call
+// from multiple goroutines.
type CollectionFileSystem interface {
http.FileSystem
+
+ // analogous to os.Stat()
Stat(name string) (os.FileInfo, error)
+
+ // analogous to os.Create(): create/truncate a file and open it O_RDWR.
Create(name string) (File, error)
+
+ // Like os.OpenFile(): create or open a file or directory.
+ //
+ // If flag&os.O_EXCL==0, it opens an existing file or
+ // directory if one exists. If flag&os.O_CREATE!=0, it creates
+ // a new empty file or directory if one does not already
+ // exist.
+ //
+ // When creating a new item, perm&os.ModeDir determines
+ // whether it is a file or a directory.
+ //
+ // A file can be opened multiple times and used concurrently
+ // from multiple goroutines. However, each File object should
+ // be used by only one goroutine at a time.
OpenFile(name string, flag int, perm os.FileMode) (File, error)
- MarshalManifest(string) (string, error)
+
+ Mkdir(name string, perm os.FileMode) error
+ Remove(name string) error
+ RemoveAll(name string) error
+ Rename(oldname, newname string) error
+ MarshalManifest(prefix string) (string, error)
}
type fileSystem struct {
}
type inode interface {
- os.FileInfo
- OpenFile(string, int, os.FileMode) (*file, error)
Parent() inode
Read([]byte, filenodePtr) (int, filenodePtr, error)
Write([]byte, filenodePtr) (int, filenodePtr, error)
Truncate(int64) error
Readdir() []os.FileInfo
+ Size() int64
Stat() os.FileInfo
sync.Locker
RLock()
// filenode implements inode.
type filenode struct {
- fileinfo
+ fileinfo fileinfo
parent *dirnode
extents []extent
repacked int64 // number of times anything in []extents has changed len
+ memsize int64 // bytes in memExtents
sync.RWMutex
}
// meaningless anyway
return
} else if ptr.off >= fn.fileinfo.size {
- ptr.off = fn.fileinfo.size
ptr.extentIdx = len(fn.extents)
ptr.extentOff = 0
ptr.repacked = fn.repacked
fn.fileinfo.size += int64(e.Len())
}
-func (fn *filenode) OpenFile(string, int, os.FileMode) (*file, error) {
- return nil, os.ErrNotExist
-}
-
func (fn *filenode) Parent() inode {
return fn.parent
}
}
func (fn *filenode) Read(p []byte, startPtr filenodePtr) (n int, ptr filenodePtr, err error) {
- fn.RLock()
- defer fn.RUnlock()
ptr = fn.seek(startPtr)
if ptr.off < 0 {
err = ErrNegativeOffset
return
}
+func (fn *filenode) Size() int64 {
+ fn.RLock()
+ defer fn.RUnlock()
+ return fn.fileinfo.Size()
+}
+
+func (fn *filenode) Stat() os.FileInfo {
+ fn.RLock()
+ defer fn.RUnlock()
+ return fn.fileinfo
+}
+
func (fn *filenode) Truncate(size int64) error {
fn.Lock()
defer fn.Unlock()
+ return fn.truncate(size)
+}
+
+func (fn *filenode) truncate(size int64) error {
+ if size == fn.fileinfo.size {
+ return nil
+ }
+ fn.repacked++
if size < fn.fileinfo.size {
- ptr := fn.seek(filenodePtr{off: size, repacked: fn.repacked - 1})
+ ptr := fn.seek(filenodePtr{off: size})
+ for i := ptr.extentIdx; i < len(fn.extents); i++ {
+ if ext, ok := fn.extents[i].(*memExtent); ok {
+ fn.memsize -= int64(ext.Len())
+ }
+ }
if ptr.extentOff == 0 {
fn.extents = fn.extents[:ptr.extentIdx]
} else {
fn.extents = fn.extents[:ptr.extentIdx+1]
- e := fn.extents[ptr.extentIdx]
- if e, ok := e.(writableExtent); ok {
- e.Truncate(ptr.extentOff)
- } else {
- fn.extents[ptr.extentIdx] = e.Slice(0, ptr.extentOff)
+ switch ext := fn.extents[ptr.extentIdx].(type) {
+ case *memExtent:
+ ext.Truncate(ptr.extentOff)
+ fn.memsize += int64(ext.Len())
+ default:
+ fn.extents[ptr.extentIdx] = ext.Slice(0, ptr.extentOff)
}
}
fn.fileinfo.size = size
- fn.repacked++
return nil
}
for size > fn.fileinfo.size {
} else if e, ok = fn.extents[len(fn.extents)-1].(writableExtent); !ok || e.Len() >= maxBlockSize {
e = &memExtent{}
fn.extents = append(fn.extents, e)
- } else {
- fn.repacked++
}
if maxgrow := int64(maxBlockSize - e.Len()); maxgrow < grow {
grow = maxgrow
}
e.Truncate(e.Len() + int(grow))
fn.fileinfo.size += grow
+ fn.memsize += grow
}
return nil
}
+// Caller must hold lock.
func (fn *filenode) Write(p []byte, startPtr filenodePtr) (n int, ptr filenodePtr, err error) {
- fn.Lock()
- defer fn.Unlock()
+ if startPtr.off > fn.fileinfo.size {
+ if err = fn.truncate(startPtr.off); err != nil {
+ return 0, startPtr, err
+ }
+ }
ptr = fn.seek(startPtr)
if ptr.off < 0 {
err = ErrNegativeOffset
prev++
e := &memExtent{}
e.Truncate(len(cando))
+ fn.memsize += int64(len(cando))
fn.extents[cur] = e
fn.extents[prev] = fn.extents[prev].Slice(0, ptr.extentOff)
ptr.extentIdx++
ptr.extentIdx--
ptr.extentOff = fn.extents[prev].Len()
fn.extents[prev].(writableExtent).Truncate(ptr.extentOff + len(cando))
+ fn.memsize += int64(len(cando))
ptr.repacked++
fn.repacked++
} else {
}
e := &memExtent{}
e.Truncate(len(cando))
+ fn.memsize += int64(len(cando))
fn.extents[cur] = e
cur++
prev++
ptr.off += int64(len(cando))
ptr.extentOff += len(cando)
+ if ptr.extentOff >= maxBlockSize {
+ fn.pruneMemExtents()
+ }
if fn.extents[ptr.extentIdx].Len() == ptr.extentOff {
ptr.extentOff = 0
ptr.extentIdx++
return
}
+// Write some data out to disk to reduce memory use. Caller must have
+// write lock.
+func (fn *filenode) pruneMemExtents() {
+ // TODO: async (don't hold Lock() while waiting for Keep)
+ // TODO: share code with (*dirnode)sync()
+ // TODO: pack/flush small blocks too, when fragmented
+ for idx, ext := range fn.extents {
+ ext, ok := ext.(*memExtent)
+ if !ok || ext.Len() < maxBlockSize {
+ continue
+ }
+ locator, _, err := fn.parent.kc.PutB(ext.buf)
+ if err != nil {
+ // TODO: stall (or return errors from)
+ // subsequent writes until flushing
+ // starts to succeed
+ continue
+ }
+ fn.memsize -= int64(ext.Len())
+ fn.extents[idx] = storedExtent{
+ kc: fn.parent.kc,
+ locator: locator,
+ size: ext.Len(),
+ offset: 0,
+ length: ext.Len(),
+ }
+ }
+}
+
// FileSystem returns a CollectionFileSystem for the collection.
-func (c *Collection) FileSystem(client *Client, kc keepClient) CollectionFileSystem {
+func (c *Collection) FileSystem(client *Client, kc keepClient) (CollectionFileSystem, error) {
fs := &fileSystem{dirnode: dirnode{
- cache: &keepBlockCache{kc: kc},
client: client,
kc: kc,
fileinfo: fileinfo{name: ".", mode: os.ModeDir | 0755},
inodes: make(map[string]inode),
}}
fs.dirnode.parent = &fs.dirnode
- fs.dirnode.loadManifest(c.ManifestText)
- return fs
+ if err := fs.dirnode.loadManifest(c.ManifestText); err != nil {
+ return nil, err
+ }
+ return fs, nil
}
type file struct {
inode
ptr filenodePtr
append bool
+ readable bool
writable bool
unreaddirs []os.FileInfo
}
func (f *file) Read(p []byte) (n int, err error) {
+ if !f.readable {
+ return 0, ErrWriteOnlyMode
+ }
+ f.inode.RLock()
+ defer f.inode.RUnlock()
n, f.ptr, err = f.inode.Read(p, f.ptr)
return
}
if ptr.off < 0 {
return f.ptr.off, ErrNegativeOffset
}
- if ptr.off > size {
- ptr.off = size
- }
if ptr.off != f.ptr.off {
f.ptr = ptr
// force filenode to recompute f.ptr fields on next
if !f.writable {
return 0, ErrReadOnlyFile
}
+ f.inode.Lock()
+ defer f.inode.Unlock()
+ if fn, ok := f.inode.(*filenode); ok && f.append {
+ f.ptr = filenodePtr{
+ off: fn.fileinfo.size,
+ extentIdx: len(fn.extents),
+ extentOff: 0,
+ repacked: fn.repacked,
+ }
+ }
n, f.ptr, err = f.inode.Write(p, f.ptr)
return
}
func (f *file) Readdir(count int) ([]os.FileInfo, error) {
- if !f.inode.IsDir() {
+ if !f.inode.Stat().IsDir() {
return nil, ErrInvalidOperation
}
if count <= 0 {
}
func (f *file) Stat() (os.FileInfo, error) {
- return f.inode, nil
+ return f.inode.Stat(), nil
}
func (f *file) Close() error {
return nil
}
-func (f *file) OpenFile(name string, flag int, perm os.FileMode) (*file, error) {
- return f.inode.OpenFile(name, flag, perm)
-}
-
type dirnode struct {
- fileinfo
- parent *dirnode
- client *Client
- kc keepClient
- cache blockCache
- inodes map[string]inode
+ fileinfo fileinfo
+ parent *dirnode
+ client *Client
+ kc keepClient
+ inodes map[string]inode
sync.RWMutex
}
-// caller must hold dn.Lock().
+// sync flushes in-memory data (for all files in the tree rooted at
+// dn) to persistent storage. Caller must hold dn.Lock().
func (dn *dirnode) sync() error {
type shortBlock struct {
fn *filenode
if len(sbs) == 0 {
return nil
}
- hash := md5.New()
- size := 0
+ block := make([]byte, 0, maxBlockSize)
for _, sb := range sbs {
- data := sb.fn.extents[sb.idx].(*memExtent).buf
- if _, err := hash.Write(data); err != nil {
- return err
- }
- size += len(data)
+ block = append(block, sb.fn.extents[sb.idx].(*memExtent).buf...)
+ }
+ locator, _, err := dn.kc.PutB(block)
+ if err != nil {
+ return err
}
- // FIXME: write to keep
- locator := fmt.Sprintf("%x+%d", hash.Sum(nil), size)
off := 0
for _, sb := range sbs {
data := sb.fn.extents[sb.idx].(*memExtent).buf
sb.fn.extents[sb.idx] = storedExtent{
- cache: dn.cache,
+ kc: dn.kc,
locator: locator,
- size: size,
+ size: len(block),
offset: off,
length: len(data),
}
off += len(data)
+ sb.fn.memsize -= int64(len(data))
}
return nil
}
func (dn *dirnode) MarshalManifest(prefix string) (string, error) {
dn.Lock()
defer dn.Unlock()
- if err := dn.sync(); err != nil {
- return "", err
- }
+ return dn.marshalManifest(prefix)
+}
+// caller must have read lock.
+func (dn *dirnode) marshalManifest(prefix string) (string, error) {
var streamLen int64
type m1segment struct {
name string
var subdirs string
var blocks []string
+ if err := dn.sync(); err != nil {
+ return "", err
+ }
+
names := make([]string, 0, len(dn.inodes))
- for name := range dn.inodes {
+ for name, node := range dn.inodes {
names = append(names, name)
+ node.Lock()
+ defer node.Unlock()
}
sort.Strings(names)
node := dn.inodes[name]
switch node := node.(type) {
case *dirnode:
- subdir, err := node.MarshalManifest(prefix + "/" + node.Name())
+ subdir, err := node.marshalManifest(prefix + "/" + name)
if err != nil {
return "", err
}
subdirs = subdirs + subdir
case *filenode:
+ if len(node.extents) == 0 {
+ segments = append(segments, m1segment{name: name})
+ break
+ }
for _, e := range node.extents {
switch e := e.(type) {
- case *memExtent:
- blocks = append(blocks, fmt.Sprintf("FIXME+%d", e.Len()))
- segments = append(segments, m1segment{
- name: node.Name(),
- offset: streamLen,
- length: int64(e.Len()),
- })
- streamLen += int64(e.Len())
case storedExtent:
if len(blocks) > 0 && blocks[len(blocks)-1] == e.locator {
streamLen -= int64(e.size)
} else {
blocks = append(blocks, e.locator)
}
- segments = append(segments, m1segment{
- name: node.Name(),
+ next := m1segment{
+ name: name,
offset: streamLen + int64(e.offset),
length: int64(e.length),
- })
+ }
+ if prev := len(segments) - 1; prev >= 0 &&
+ segments[prev].name == name &&
+ segments[prev].offset+segments[prev].length == next.offset {
+ segments[prev].length += next.length
+ } else {
+ segments = append(segments, next)
+ }
streamLen += int64(e.size)
default:
+ // This can't happen: we
+ // haven't unlocked since
+ // calling sync().
panic(fmt.Sprintf("can't marshal extent type %T", e))
}
}
}
var filetokens []string
for _, s := range segments {
- filetokens = append(filetokens, fmt.Sprintf("%d:%d:%s", s.offset, s.length, s.name))
+ filetokens = append(filetokens, fmt.Sprintf("%d:%d:%s", s.offset, s.length, manifestEscape(s.name)))
}
if len(filetokens) == 0 {
return subdirs, nil
} else if len(blocks) == 0 {
blocks = []string{"d41d8cd98f00b204e9800998ecf8427e+0"}
}
- return prefix + " " + strings.Join(blocks, " ") + " " + strings.Join(filetokens, " ") + "\n" + subdirs, nil
+ return manifestEscape(prefix) + " " + strings.Join(blocks, " ") + " " + strings.Join(filetokens, " ") + "\n" + subdirs, nil
}
-func (dn *dirnode) loadManifest(txt string) {
+func (dn *dirnode) loadManifest(txt string) error {
// FIXME: faster
var dirname string
- for _, stream := range strings.Split(txt, "\n") {
+ streams := strings.Split(txt, "\n")
+ if streams[len(streams)-1] != "" {
+ return fmt.Errorf("line %d: no trailing newline", len(streams))
+ }
+ for i, stream := range streams[:len(streams)-1] {
+ lineno := i + 1
var extents []storedExtent
+ var anyFileTokens bool
+ var pos int64
+ var extIdx int
for i, token := range strings.Split(stream, " ") {
if i == 0 {
dirname = manifestUnescape(token)
continue
}
if !strings.Contains(token, ":") {
+ if anyFileTokens {
+ return fmt.Errorf("line %d: bad file segment %q", lineno, token)
+ }
toks := strings.SplitN(token, "+", 3)
if len(toks) < 2 {
- // FIXME: broken
- continue
+ return fmt.Errorf("line %d: bad locator %q", lineno, token)
}
length, err := strconv.ParseInt(toks[1], 10, 32)
if err != nil || length < 0 {
- // FIXME: broken
- continue
+ return fmt.Errorf("line %d: bad locator %q", lineno, token)
}
extents = append(extents, storedExtent{
locator: token,
length: int(length),
})
continue
+ } else if len(extents) == 0 {
+ return fmt.Errorf("line %d: bad locator %q", lineno, token)
}
+
toks := strings.Split(token, ":")
if len(toks) != 3 {
- // FIXME: broken manifest
- continue
+ return fmt.Errorf("line %d: bad file segment %q", lineno, token)
}
+ anyFileTokens = true
+
offset, err := strconv.ParseInt(toks[0], 10, 64)
if err != nil || offset < 0 {
- // FIXME: broken manifest
- continue
+ return fmt.Errorf("line %d: bad file segment %q", lineno, token)
}
length, err := strconv.ParseInt(toks[1], 10, 64)
if err != nil || length < 0 {
- // FIXME: broken manifest
- continue
+ return fmt.Errorf("line %d: bad file segment %q", lineno, token)
}
name := path.Clean(dirname + "/" + manifestUnescape(toks[2]))
- dn.makeParentDirs(name)
- f, err := dn.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0700)
+ fnode, err := dn.createFileAndParents(name)
if err != nil {
- // FIXME: broken
- continue
- }
- if f.inode.Stat().IsDir() {
- f.Close()
- // FIXME: broken manifest
- continue
+ return fmt.Errorf("line %d: cannot use path %q: %s", lineno, name, err)
}
// Map the stream offset/range coordinates to
// block/offset/range coordinates and add
// corresponding storedExtents to the filenode
- var pos int64
- for _, e := range extents {
- next := pos + int64(e.Len())
- if next < offset {
+ if pos > offset {
+ // Can't continue where we left off.
+ // TODO: binary search instead of
+ // rewinding all the way (but this
+ // situation might be rare anyway)
+ extIdx, pos = 0, 0
+ }
+ for next := int64(0); extIdx < len(extents); extIdx, pos = extIdx+1, next {
+ e := extents[extIdx]
+ next = pos + int64(e.Len())
+ if next <= offset || e.Len() == 0 {
pos = next
continue
}
- if pos > offset+length {
+ if pos >= offset+length {
break
}
var blkOff int
if pos+int64(blkOff+blkLen) > offset+length {
blkLen = int(offset + length - pos - int64(blkOff))
}
- f.inode.(*filenode).appendExtent(storedExtent{
- cache: dn.cache,
+ fnode.appendExtent(storedExtent{
+ kc: dn.kc,
locator: e.locator,
size: e.size,
offset: blkOff,
length: blkLen,
})
- pos = next
+ if next > offset+length {
+ break
+ }
+ }
+ if extIdx == len(extents) && pos < offset+length {
+ return fmt.Errorf("line %d: invalid segment in %d-byte stream: %q", lineno, pos, token)
}
- f.Close()
+ }
+ if !anyFileTokens {
+ return fmt.Errorf("line %d: no file segments", lineno)
+ } else if len(extents) == 0 {
+ return fmt.Errorf("line %d: no locators", lineno)
+ } else if dirname == "" {
+ return fmt.Errorf("line %d: no stream name", lineno)
}
}
+ return nil
}
-func (dn *dirnode) makeParentDirs(name string) {
- names := strings.Split(name, "/")
- for _, name := range names[:len(names)-1] {
- dn.Lock()
- defer dn.Unlock()
- if n, ok := dn.inodes[name]; !ok {
- n := &dirnode{
- parent: dn,
- client: dn.client,
- kc: dn.kc,
- fileinfo: fileinfo{
- name: name,
- mode: os.ModeDir | 0755,
- },
+// only safe to call from loadManifest -- no locking
+func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
+ names := strings.Split(path, "/")
+ if basename := names[len(names)-1]; basename == "" || basename == "." || basename == ".." {
+ err = fmt.Errorf("invalid filename")
+ return
+ }
+ var node inode = dn
+ for i, name := range names {
+ dn, ok := node.(*dirnode)
+ if !ok {
+ err = ErrFileExists
+ return
+ }
+ if name == "" || name == "." {
+ continue
+ }
+ if name == ".." {
+ node = dn.parent
+ continue
+ }
+ node, ok = dn.inodes[name]
+ if !ok {
+ if i == len(names)-1 {
+ fn = dn.newFilenode(name, 0755)
+ return
}
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
+ node = dn.newDirnode(name, 0755)
+ }
+ }
+ var ok bool
+ if fn, ok = node.(*filenode); !ok {
+ err = ErrInvalidArgument
+ }
+ return
+}
+
+func (dn *dirnode) mkdir(name string) (*file, error) {
+ return dn.OpenFile(name, os.O_CREATE|os.O_EXCL, os.ModeDir|0755)
+}
+
+func (dn *dirnode) Mkdir(name string, perm os.FileMode) error {
+ f, err := dn.mkdir(name)
+ if err == nil {
+ err = f.Close()
+ }
+ return err
+}
+
+func (dn *dirnode) Remove(name string) error {
+ return dn.remove(name, false)
+}
+
+func (dn *dirnode) RemoveAll(name string) error {
+ return dn.remove(name, true)
+}
+
+func (dn *dirnode) remove(name string, recursive bool) error {
+ dirname, name := path.Split(name)
+ if name == "" || name == "." || name == ".." {
+ return ErrInvalidArgument
+ }
+ dn, ok := dn.lookupPath(dirname).(*dirnode)
+ if !ok {
+ return os.ErrNotExist
+ }
+ dn.Lock()
+ defer dn.Unlock()
+ switch node := dn.inodes[name].(type) {
+ case nil:
+ return os.ErrNotExist
+ case *dirnode:
+ node.RLock()
+ defer node.RUnlock()
+ if !recursive && len(node.inodes) > 0 {
+ return ErrDirectoryNotEmpty
+ }
+ }
+ delete(dn.inodes, name)
+ return nil
+}
+
+func (dn *dirnode) Rename(oldname, newname string) error {
+ olddir, oldname := path.Split(oldname)
+ if oldname == "" || oldname == "." || oldname == ".." {
+ return ErrInvalidArgument
+ }
+ olddirf, err := dn.OpenFile(olddir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", olddir, err)
+ }
+ defer olddirf.Close()
+ newdir, newname := path.Split(newname)
+ if newname == "." || newname == ".." {
+ return ErrInvalidArgument
+ } else if newname == "" {
+ // Rename("a/b", "c/") means Rename("a/b", "c/b")
+ newname = oldname
+ }
+ newdirf, err := dn.OpenFile(newdir+".", os.O_RDONLY, 0)
+ if err != nil {
+ return fmt.Errorf("%q: %s", newdir, err)
+ }
+ defer newdirf.Close()
+
+ // When acquiring locks on multiple nodes, all common
+ // ancestors must be locked first in order to avoid
+ // deadlock. This is assured by locking the path from root to
+ // newdir, then locking the path from root to olddir, skipping
+ // any already-locked nodes.
+ needLock := []sync.Locker{}
+ for _, f := range []*file{olddirf, newdirf} {
+ node := f.inode
+ needLock = append(needLock, node)
+ for node.Parent() != node {
+ node = node.Parent()
+ needLock = append(needLock, node)
+ }
+ }
+ locked := map[sync.Locker]bool{}
+ for i := len(needLock) - 1; i >= 0; i-- {
+ if n := needLock[i]; !locked[n] {
+ n.Lock()
+ defer n.Unlock()
+ locked[n] = true
+ }
+ }
+
+ olddn := olddirf.inode.(*dirnode)
+ newdn := newdirf.inode.(*dirnode)
+ oldinode, ok := olddn.inodes[oldname]
+ if !ok {
+ return os.ErrNotExist
+ }
+ if existing, ok := newdn.inodes[newname]; ok {
+ // overwriting an existing file or dir
+ if dn, ok := existing.(*dirnode); ok {
+ if !oldinode.Stat().IsDir() {
+ return ErrIsDirectory
+ }
+ dn.RLock()
+ defer dn.RUnlock()
+ if len(dn.inodes) > 0 {
+ return ErrDirectoryNotEmpty
}
- dn.inodes[name] = n
- dn.fileinfo.size++
- dn = n
- } else if n, ok := n.(*dirnode); ok {
- dn = n
- } else {
- // fail
- return
}
+ } else {
+ newdn.fileinfo.size++
}
+ newdn.inodes[newname] = oldinode
+ delete(olddn.inodes, oldname)
+ olddn.fileinfo.size--
+ return nil
}
func (dn *dirnode) Parent() inode {
+ dn.RLock()
+ defer dn.RUnlock()
return dn.parent
}
return 0, ptr, ErrInvalidOperation
}
+func (dn *dirnode) Size() int64 {
+ dn.RLock()
+ defer dn.RUnlock()
+ return dn.fileinfo.Size()
+}
+
+func (dn *dirnode) Stat() os.FileInfo {
+ dn.RLock()
+ defer dn.RUnlock()
+ return dn.fileinfo
+}
+
func (dn *dirnode) Truncate(int64) error {
return ErrInvalidOperation
}
-func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*file, error) {
- name = strings.TrimSuffix(name, "/")
- if name == "." || name == "" {
- return &file{inode: dn}, nil
- }
- if dirname, name := path.Split(name); dirname != "" {
- // OpenFile("foo/bar/baz") =>
- // OpenFile("foo/bar").OpenFile("baz") (or
- // ErrNotExist, if foo/bar is a file)
- f, err := dn.OpenFile(dirname, os.O_RDONLY, 0)
- if err != nil {
- return nil, err
+// lookupPath returns the inode for the file/directory with the given
+// name (which may contain "/" separators), along with its parent
+// node. If no such file/directory exists, the returned node is nil.
+func (dn *dirnode) lookupPath(path string) (node inode) {
+ node = dn
+ for _, name := range strings.Split(path, "/") {
+ dn, ok := node.(*dirnode)
+ if !ok {
+ return nil
}
- defer f.Close()
- if dn, ok := f.inode.(*dirnode); ok {
- return dn.OpenFile(name, flag, perm)
- } else {
- return nil, os.ErrNotExist
+ if name == "." || name == "" {
+ continue
}
+ if name == ".." {
+ node = node.Parent()
+ continue
+ }
+ dn.RLock()
+ node = dn.inodes[name]
+ dn.RUnlock()
}
- dn.Lock()
- defer dn.Unlock()
- if name == ".." {
- return &file{inode: dn.parent}, nil
+ return
+}
+
+func (dn *dirnode) newDirnode(name string, perm os.FileMode) *dirnode {
+ child := &dirnode{
+ parent: dn,
+ client: dn.client,
+ kc: dn.kc,
+ fileinfo: fileinfo{
+ name: name,
+ mode: os.ModeDir | perm,
+ },
+ }
+ if dn.inodes == nil {
+ dn.inodes = make(map[string]inode)
+ }
+ dn.inodes[name] = child
+ dn.fileinfo.size++
+ return child
+}
+
+func (dn *dirnode) newFilenode(name string, perm os.FileMode) *filenode {
+ child := &filenode{
+ parent: dn,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm,
+ },
+ }
+ if dn.inodes == nil {
+ dn.inodes = make(map[string]inode)
+ }
+ dn.inodes[name] = child
+ dn.fileinfo.size++
+ return child
+}
+
+// OpenFile is analogous to os.OpenFile().
+func (dn *dirnode) OpenFile(name string, flag int, perm os.FileMode) (*file, error) {
+ if flag&os.O_SYNC != 0 {
+ return nil, ErrSyncNotSupported
+ }
+ dirname, name := path.Split(name)
+ dn, ok := dn.lookupPath(dirname).(*dirnode)
+ if !ok {
+ return nil, os.ErrNotExist
+ }
+ var readable, writable bool
+ switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
+ case os.O_RDWR:
+ readable = true
+ writable = true
+ case os.O_RDONLY:
+ readable = true
+ case os.O_WRONLY:
+ writable = true
+ default:
+ return nil, fmt.Errorf("invalid flags 0x%x", flag)
+ }
+ if !writable {
+ // A directory can be opened via "foo/", "foo/.", or
+ // "foo/..".
+ switch name {
+ case ".", "":
+ return &file{inode: dn}, nil
+ case "..":
+ return &file{inode: dn.Parent()}, nil
+ }
+ }
+ createMode := flag&os.O_CREATE != 0
+ if createMode {
+ dn.Lock()
+ defer dn.Unlock()
+ } else {
+ dn.RLock()
+ defer dn.RUnlock()
}
n, ok := dn.inodes[name]
if !ok {
- if flag&os.O_CREATE == 0 {
+ if !createMode {
return nil, os.ErrNotExist
}
- n = &filenode{
- parent: dn,
- fileinfo: fileinfo{
- name: name,
- mode: 0755,
- },
- }
- if dn.inodes == nil {
- dn.inodes = make(map[string]inode)
+ if perm.IsDir() {
+ n = dn.newDirnode(name, 0755)
+ } else {
+ n = dn.newFilenode(name, 0755)
}
- dn.inodes[name] = n
- dn.fileinfo.size++
} else if flag&os.O_EXCL != 0 {
return nil, ErrFileExists
+ } else if flag&os.O_TRUNC != 0 {
+ if !writable {
+ return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
+ } else if fn, ok := n.(*filenode); !ok {
+ return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
+ } else {
+ fn.Truncate(0)
+ }
}
return &file{
inode: n,
append: flag&os.O_APPEND != 0,
- writable: flag&(os.O_WRONLY|os.O_RDWR) != 0,
+ readable: readable,
+ writable: writable,
}, nil
}
}
type storedExtent struct {
- cache blockCache
+ kc keepClient
locator string
size int
offset int
maxlen := se.length - int(off)
if len(p) > maxlen {
p = p[:maxlen]
- n, err = se.cache.ReadAt(se.locator, p, int(off)+se.offset)
+ n, err = se.kc.ReadAt(se.locator, p, int(off)+se.offset)
if err == nil {
err = io.EOF
}
return
}
- return se.cache.ReadAt(se.locator, p, int(off)+se.offset)
-}
-
-type blockCache interface {
- ReadAt(locator string, p []byte, off int) (n int, err error)
-}
-
-type keepBlockCache struct {
- kc keepClient
-}
-
-var scratch = make([]byte, 2<<26)
-
-func (kbc *keepBlockCache) ReadAt(locator string, p []byte, off int) (int, error) {
- return kbc.kc.ReadAt(locator, p, off)
+ return se.kc.ReadAt(se.locator, p, int(off)+se.offset)
}
func canonicalName(name string) string {
var manifestEscapeSeq = regexp.MustCompile(`\\([0-9]{3}|\\)`)
-func manifestUnescapeSeq(seq string) string {
+func manifestUnescapeFunc(seq string) string {
if seq == `\\` {
return `\`
}
}
func manifestUnescape(s string) string {
- return manifestEscapeSeq.ReplaceAllStringFunc(s, manifestUnescapeSeq)
+ return manifestEscapeSeq.ReplaceAllStringFunc(s, manifestUnescapeFunc)
+}
+
+var manifestEscapedChar = regexp.MustCompile(`[^\.\w/]`)
+
+func manifestEscapeFunc(seq string) string {
+ return fmt.Sprintf("\\%03o", byte(seq[0]))
+}
+
+func manifestEscape(s string) string {
+ return manifestEscapedChar.ReplaceAllStringFunc(s, manifestEscapeFunc)
}