1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: Apache-2.0
21 ErrReadOnlyFile = errors.New("read-only file")
22 ErrNegativeOffset = errors.New("cannot seek to negative offset")
23 ErrFileExists = errors.New("file exists")
24 ErrInvalidOperation = errors.New("invalid operation")
25 ErrInvalidArgument = errors.New("invalid argument")
26 ErrDirectoryNotEmpty = errors.New("directory not empty")
27 ErrWriteOnlyMode = errors.New("file is O_WRONLY")
28 ErrSyncNotSupported = errors.New("O_SYNC flag is not supported")
29 ErrIsDirectory = errors.New("cannot rename file to overwrite existing directory")
30 ErrNotADirectory = errors.New("not a directory")
31 ErrPermission = os.ErrPermission
34 type syncer interface {
38 // A File is an *os.File-like interface for reading and writing files
46 Readdir(int) ([]os.FileInfo, error)
47 Stat() (os.FileInfo, error)
52 // A FileSystem is an http.Filesystem plus Stat() and support for
53 // opening writable files. All methods are safe to call from multiple
55 type FileSystem interface {
61 // filesystem-wide lock: used by Rename() to prevent deadlock
62 // while locking multiple inodes.
65 // throttle for limiting concurrent background writers
68 // create a new node with nil parent.
69 newNode(name string, perm os.FileMode, modTime time.Time) (node inode, err error)
71 // analogous to os.Stat()
72 Stat(name string) (os.FileInfo, error)
74 // analogous to os.Create(): create/truncate a file and open it O_RDWR.
75 Create(name string) (File, error)
77 // Like os.OpenFile(): create or open a file or directory.
79 // If flag&os.O_EXCL==0, it opens an existing file or
80 // directory if one exists. If flag&os.O_CREATE!=0, it creates
81 // a new empty file or directory if one does not already
84 // When creating a new item, perm&os.ModeDir determines
85 // whether it is a file or a directory.
87 // A file can be opened multiple times and used concurrently
88 // from multiple goroutines. However, each File object should
89 // be used by only one goroutine at a time.
90 OpenFile(name string, flag int, perm os.FileMode) (File, error)
92 Mkdir(name string, perm os.FileMode) error
93 Remove(name string) error
94 RemoveAll(name string) error
95 Rename(oldname, newname string) error
97 // Write buffered data from memory to storage, returning when
98 // all updates have been saved to persistent storage.
101 // Write buffered data from memory to storage, but don't wait
102 // for all writes to finish before returning. If shortBlocks
103 // is true, flush everything; otherwise, if there's less than
104 // a full block of buffered data at the end of a stream, leave
105 // it buffered in memory in case more data can be appended. If
106 // path is "", flush all dirs/streams; otherwise, flush only
107 // the specified dir/stream.
108 Flush(path string, shortBlocks bool) error
111 type inode interface {
112 SetParent(parent inode, name string)
115 Read([]byte, filenodePtr) (int, filenodePtr, error)
116 Write([]byte, filenodePtr) (int, filenodePtr, error)
117 Truncate(int64) error
119 Readdir() ([]os.FileInfo, error)
121 FileInfo() os.FileInfo
123 // Child() performs lookups and updates of named child nodes.
125 // (The term "child" here is used strictly. This means name is
126 // not "." or "..", and name does not contain "/".)
128 // If replace is non-nil, Child calls replace(x) where x is
129 // the current child inode with the given name. If possible,
130 // the child inode is replaced with the one returned by
133 // If replace(x) returns an inode (besides x or nil) that is
134 // subsequently returned by Child(), then Child()'s caller
135 // must ensure the new child's name and parent are set/updated
136 // to Child()'s name argument and its receiver respectively.
137 // This is not necessarily done before replace(x) returns, but
138 // it must be done before Child()'s caller releases the
141 // Nil represents "no child". replace(nil) signifies that no
142 // child with this name exists yet. If replace() returns nil,
143 // the existing child should be deleted if possible.
145 // An implementation of Child() is permitted to ignore
146 // replace() or its return value. For example, a regular file
147 // inode does not have children, so Child() always returns
150 // Child() returns the child, if any, with the given name: if
151 // a child was added or changed, the new child is returned.
153 // Caller must have lock (or rlock if replace is nil).
154 Child(name string, replace func(inode) (inode, error)) (inode, error)
161 type fileinfo struct {
168 // Name implements os.FileInfo.
169 func (fi fileinfo) Name() string {
173 // ModTime implements os.FileInfo.
174 func (fi fileinfo) ModTime() time.Time {
178 // Mode implements os.FileInfo.
179 func (fi fileinfo) Mode() os.FileMode {
183 // IsDir implements os.FileInfo.
184 func (fi fileinfo) IsDir() bool {
185 return fi.mode&os.ModeDir != 0
188 // Size implements os.FileInfo.
189 func (fi fileinfo) Size() int64 {
193 // Sys implements os.FileInfo.
194 func (fi fileinfo) Sys() interface{} {
198 type nullnode struct{}
200 func (*nullnode) Mkdir(string, os.FileMode) error {
201 return ErrInvalidOperation
204 func (*nullnode) Read([]byte, filenodePtr) (int, filenodePtr, error) {
205 return 0, filenodePtr{}, ErrInvalidOperation
208 func (*nullnode) Write([]byte, filenodePtr) (int, filenodePtr, error) {
209 return 0, filenodePtr{}, ErrInvalidOperation
212 func (*nullnode) Truncate(int64) error {
213 return ErrInvalidOperation
216 func (*nullnode) FileInfo() os.FileInfo {
220 func (*nullnode) IsDir() bool {
224 func (*nullnode) Readdir() ([]os.FileInfo, error) {
225 return nil, ErrInvalidOperation
228 func (*nullnode) Child(name string, replace func(inode) (inode, error)) (inode, error) {
229 return nil, ErrNotADirectory
232 type treenode struct {
235 inodes map[string]inode
241 func (n *treenode) FS() FileSystem {
245 func (n *treenode) SetParent(p inode, name string) {
249 n.fileinfo.name = name
252 func (n *treenode) Parent() inode {
258 func (n *treenode) IsDir() bool {
262 func (n *treenode) Child(name string, replace func(inode) (inode, error)) (child inode, err error) {
263 child = n.inodes[name]
264 if name == "" || name == "." || name == ".." {
265 err = ErrInvalidArgument
271 newchild, err := replace(child)
276 delete(n.inodes, name)
277 } else if newchild != child {
278 n.inodes[name] = newchild
279 n.fileinfo.modTime = time.Now()
285 func (n *treenode) Size() int64 {
286 return n.FileInfo().Size()
289 func (n *treenode) FileInfo() os.FileInfo {
292 n.fileinfo.size = int64(len(n.inodes))
296 func (n *treenode) Readdir() (fi []os.FileInfo, err error) {
299 fi = make([]os.FileInfo, 0, len(n.inodes))
300 for _, inode := range n.inodes {
301 fi = append(fi, inode.FileInfo())
306 func (n *treenode) Sync() error {
309 for _, inode := range n.inodes {
310 syncer, ok := inode.(syncer)
312 return ErrInvalidOperation
322 type fileSystem struct {
329 func (fs *fileSystem) rootnode() inode {
333 func (fs *fileSystem) throttle() *throttle {
337 func (fs *fileSystem) locker() sync.Locker {
341 // OpenFile is analogous to os.OpenFile().
342 func (fs *fileSystem) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
343 return fs.openFile(name, flag, perm)
346 func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*filehandle, error) {
347 if flag&os.O_SYNC != 0 {
348 return nil, ErrSyncNotSupported
350 dirname, name := path.Split(name)
351 parent, err := rlookup(fs.root, dirname)
355 var readable, writable bool
356 switch flag & (os.O_RDWR | os.O_RDONLY | os.O_WRONLY) {
365 return nil, fmt.Errorf("invalid flags 0x%x", flag)
367 if !writable && parent.IsDir() {
368 // A directory can be opened via "foo/", "foo/.", or
372 return &filehandle{inode: parent}, nil
374 return &filehandle{inode: parent.Parent()}, nil
377 createMode := flag&os.O_CREATE != 0
380 defer parent.Unlock()
383 defer parent.RUnlock()
385 n, err := parent.Child(name, nil)
390 return nil, os.ErrNotExist
392 n, err = parent.Child(name, func(inode) (repl inode, err error) {
393 repl, err = parent.FS().newNode(name, perm|0755, time.Now())
397 repl.SetParent(parent, name)
403 // Parent rejected new child, but returned no error
404 return nil, ErrInvalidArgument
406 } else if flag&os.O_EXCL != 0 {
407 return nil, ErrFileExists
408 } else if flag&os.O_TRUNC != 0 {
410 return nil, fmt.Errorf("invalid flag O_TRUNC in read-only mode")
411 } else if n.IsDir() {
412 return nil, fmt.Errorf("invalid flag O_TRUNC when opening directory")
413 } else if err := n.Truncate(0); err != nil {
419 append: flag&os.O_APPEND != 0,
425 func (fs *fileSystem) Open(name string) (http.File, error) {
426 return fs.OpenFile(name, os.O_RDONLY, 0)
429 func (fs *fileSystem) Create(name string) (File, error) {
430 return fs.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0)
433 func (fs *fileSystem) Mkdir(name string, perm os.FileMode) error {
434 dirname, name := path.Split(name)
435 n, err := rlookup(fs.root, dirname)
441 if child, err := n.Child(name, nil); err != nil {
443 } else if child != nil {
447 _, err = n.Child(name, func(inode) (repl inode, err error) {
448 repl, err = n.FS().newNode(name, perm|os.ModeDir, time.Now())
452 repl.SetParent(n, name)
458 func (fs *fileSystem) Stat(name string) (os.FileInfo, error) {
459 node, err := rlookup(fs.root, name)
463 return node.FileInfo(), nil
466 func (fs *fileSystem) Rename(oldname, newname string) error {
467 olddir, oldname := path.Split(oldname)
468 if oldname == "" || oldname == "." || oldname == ".." {
469 return ErrInvalidArgument
471 olddirf, err := fs.openFile(olddir+".", os.O_RDONLY, 0)
473 return fmt.Errorf("%q: %s", olddir, err)
475 defer olddirf.Close()
477 newdir, newname := path.Split(newname)
478 if newname == "." || newname == ".." {
479 return ErrInvalidArgument
480 } else if newname == "" {
481 // Rename("a/b", "c/") means Rename("a/b", "c/b")
484 newdirf, err := fs.openFile(newdir+".", os.O_RDONLY, 0)
486 return fmt.Errorf("%q: %s", newdir, err)
488 defer newdirf.Close()
490 // TODO: If the nearest common ancestor ("nca") of olddirf and
491 // newdirf is on a different filesystem than fs, we should
492 // call nca.FS().Rename() instead of proceeding. Until then
493 // it's awkward for filesystems to implement their own Rename
494 // methods effectively: the only one that runs is the one on
495 // the root FileSystem exposed to the caller (webdav, fuse,
498 // When acquiring locks on multiple inodes, avoid deadlock by
499 // locking the entire containing filesystem first.
500 cfs := olddirf.inode.FS()
502 defer cfs.locker().Unlock()
504 if cfs != newdirf.inode.FS() {
505 // Moving inodes across filesystems is not (yet)
506 // supported. Locking inodes from different
507 // filesystems could deadlock, so we must error out
509 return ErrInvalidArgument
512 // To ensure we can test reliably whether we're about to move
513 // a directory into itself, lock all potential common
514 // ancestors of olddir and newdir.
515 needLock := []sync.Locker{}
516 for _, node := range []inode{olddirf.inode, newdirf.inode} {
517 needLock = append(needLock, node)
518 for node.Parent() != node && node.Parent().FS() == node.FS() {
520 needLock = append(needLock, node)
523 locked := map[sync.Locker]bool{}
524 for i := len(needLock) - 1; i >= 0; i-- {
525 if n := needLock[i]; !locked[n] {
532 _, err = olddirf.inode.Child(oldname, func(oldinode inode) (inode, error) {
534 return oldinode, os.ErrNotExist
536 if locked[oldinode] {
537 // oldinode cannot become a descendant of itself.
538 return oldinode, ErrInvalidArgument
540 if oldinode.FS() != cfs && newdirf.inode != olddirf.inode {
541 // moving a mount point to a different parent
542 // is not (yet) supported.
543 return oldinode, ErrInvalidArgument
545 accepted, err := newdirf.inode.Child(newname, func(existing inode) (inode, error) {
546 if existing != nil && existing.IsDir() {
547 return existing, ErrIsDirectory
552 // Leave oldinode in olddir.
555 accepted.SetParent(newdirf.inode, newname)
561 func (fs *fileSystem) Remove(name string) error {
562 return fs.remove(strings.TrimRight(name, "/"), false)
565 func (fs *fileSystem) RemoveAll(name string) error {
566 err := fs.remove(strings.TrimRight(name, "/"), true)
567 if os.IsNotExist(err) {
568 // "If the path does not exist, RemoveAll returns
569 // nil." (see "os" pkg)
575 func (fs *fileSystem) remove(name string, recursive bool) error {
576 dirname, name := path.Split(name)
577 if name == "" || name == "." || name == ".." {
578 return ErrInvalidArgument
580 dir, err := rlookup(fs.root, dirname)
586 _, err = dir.Child(name, func(node inode) (inode, error) {
588 return nil, os.ErrNotExist
590 if !recursive && node.IsDir() && node.Size() > 0 {
591 return node, ErrDirectoryNotEmpty
598 func (fs *fileSystem) Sync() error {
599 if syncer, ok := fs.root.(syncer); ok {
602 return ErrInvalidOperation
605 func (fs *fileSystem) Flush(string, bool) error {
606 log.Printf("TODO: flush fileSystem")
607 return ErrInvalidOperation
610 // rlookup (recursive lookup) returns the inode for the file/directory
611 // with the given name (which may contain "/" separators). If no such
612 // file/directory exists, the returned node is nil.
613 func rlookup(start inode, path string) (node inode, err error) {
615 for _, name := range strings.Split(path, "/") {
617 if name == "." || name == "" {
625 node, err = func() (inode, error) {
628 return node.Child(name, nil)
630 if node == nil || err != nil {
634 if node == nil && err == nil {
640 func permittedName(name string) bool {
641 return name != "" && name != "." && name != ".." && !strings.Contains(name, "/")