projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
17778: Merge branch 'master' into 17778-doc-update
[arvados.git]
/
sdk
/
go
/
arvados
/
fs_collection.go
diff --git
a/sdk/go/arvados/fs_collection.go
b/sdk/go/arvados/fs_collection.go
index 0edc48162b1a031b8487ac9b38997c0a4b6f3f4d..b743ab368e33f69a5c1710d63dc410af8a380ffc 100644
(file)
--- a/
sdk/go/arvados/fs_collection.go
+++ b/
sdk/go/arvados/fs_collection.go
@@
-38,9
+38,6
@@
type CollectionFileSystem interface {
// Total data bytes in all files.
Size() int64
// Total data bytes in all files.
Size() int64
-
- // Memory consumed by buffered file data.
- memorySize() int64
}
type collectionFileSystem struct {
}
type collectionFileSystem struct {
@@
-109,16
+106,15
@@
func (fs *collectionFileSystem) newNode(name string, perm os.FileMode, modTime t
inodes: make(map[string]inode),
},
}, nil
inodes: make(map[string]inode),
},
}, nil
- } else {
- return &filenode{
- fs: fs,
- fileinfo: fileinfo{
- name: name,
- mode: perm & ^os.ModeDir,
- modTime: modTime,
- },
- }, nil
}
}
+ return &filenode{
+ fs: fs,
+ fileinfo: fileinfo{
+ name: name,
+ mode: perm & ^os.ModeDir,
+ modTime: modTime,
+ },
+ }, nil
}
func (fs *collectionFileSystem) Child(name string, replace func(inode) (inode, error)) (inode, error) {
}
func (fs *collectionFileSystem) Child(name string, replace func(inode) (inode, error)) (inode, error) {
@@
-233,10
+229,10
@@
func (fs *collectionFileSystem) Flush(path string, shortBlocks bool) error {
return dn.flush(context.TODO(), names, flushOpts{sync: false, shortBlocks: shortBlocks})
}
return dn.flush(context.TODO(), names, flushOpts{sync: false, shortBlocks: shortBlocks})
}
-func (fs *collectionFileSystem)
m
emorySize() int64 {
+func (fs *collectionFileSystem)
M
emorySize() int64 {
fs.fileSystem.root.Lock()
defer fs.fileSystem.root.Unlock()
fs.fileSystem.root.Lock()
defer fs.fileSystem.root.Unlock()
- return fs.fileSystem.root.(*dirnode).
m
emorySize()
+ return fs.fileSystem.root.(*dirnode).
M
emorySize()
}
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
}
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
@@
-568,8
+564,6
@@
func (fn *filenode) Write(p []byte, startPtr filenodePtr) (n int, ptr filenodePt
seg.Truncate(len(cando))
fn.memsize += int64(len(cando))
fn.segments[cur] = seg
seg.Truncate(len(cando))
fn.memsize += int64(len(cando))
fn.segments[cur] = seg
- cur++
- prev++
}
}
}
}
@@
-680,6
+674,7
@@
func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode
if err != nil {
return nil, err
}
if err != nil {
return nil, err
}
+ coll.UUID = dn.fs.uuid
data, err := json.Marshal(&coll)
if err == nil {
data = append(data, '\n')
data, err := json.Marshal(&coll)
if err == nil {
data = append(data, '\n')
@@
-731,12
+726,11
@@
func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
// it fails, we'll try again next time.
close(done)
return nil
// it fails, we'll try again next time.
close(done)
return nil
- } else {
- // In sync mode, we proceed regardless of
- // whether another flush is in progress: It
- // can't finish before we do, because we hold
- // fn's lock until we finish our own writes.
}
}
+ // In sync mode, we proceed regardless of
+ // whether another flush is in progress: It
+ // can't finish before we do, because we hold
+ // fn's lock until we finish our own writes.
seg.flushing = done
offsets = append(offsets, len(block))
if len(refs) == 1 {
seg.flushing = done
offsets = append(offsets, len(block))
if len(refs) == 1 {
@@
-804,9
+798,8
@@
func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
}()
if sync {
return <-errs
}()
if sync {
return <-errs
- } else {
- return nil
}
}
+ return nil
}
type flushOpts struct {
}
type flushOpts struct {
@@
-884,14
+877,14
@@
func (dn *dirnode) flush(ctx context.Context, names []string, opts flushOpts) er
}
// caller must have write lock.
}
// caller must have write lock.
-func (dn *dirnode)
m
emorySize() (size int64) {
+func (dn *dirnode)
M
emorySize() (size int64) {
for _, name := range dn.sortedNames() {
node := dn.inodes[name]
node.Lock()
defer node.Unlock()
switch node := node.(type) {
case *dirnode:
for _, name := range dn.sortedNames() {
node := dn.inodes[name]
node.Lock()
defer node.Unlock()
switch node := node.(type) {
case *dirnode:
- size += node.
m
emorySize()
+ size += node.
M
emorySize()
case *filenode:
for _, seg := range node.segments {
switch seg := seg.(type) {
case *filenode:
for _, seg := range node.segments {
switch seg := seg.(type) {
@@
-1109,9
+1102,9
@@
func (dn *dirnode) loadManifest(txt string) error {
// situation might be rare anyway)
segIdx, pos = 0, 0
}
// situation might be rare anyway)
segIdx, pos = 0, 0
}
- for
next := int64(0)
; segIdx < len(segments); segIdx++ {
+ for ; segIdx < len(segments); segIdx++ {
seg := segments[segIdx]
seg := segments[segIdx]
- next = pos + int64(seg.Len())
+ next
:
= pos + int64(seg.Len())
if next <= offset || seg.Len() == 0 {
pos = next
continue
if next <= offset || seg.Len() == 0 {
pos = next
continue
@@
-1175,9
+1168,12
@@
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
node = node.Parent()
continue
}
node = node.Parent()
continue
}
+ modtime := node.Parent().FileInfo().ModTime()
+ node.Lock()
+ locked := node
node, err = node.Child(name, func(child inode) (inode, error) {
if child == nil {
node, err = node.Child(name, func(child inode) (inode, error) {
if child == nil {
- child, err := node.FS().newNode(name, 0755|os.ModeDir,
node.Parent().FileInfo().ModTime()
)
+ child, err := node.FS().newNode(name, 0755|os.ModeDir,
modtime
)
if err != nil {
return nil, err
}
if err != nil {
return nil, err
}
@@
-1189,6
+1185,7
@@
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
return child, nil
}
})
return child, nil
}
})
+ locked.Unlock()
if err != nil {
return
}
if err != nil {
return
}
@@
-1199,10
+1196,13
@@
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
err = fmt.Errorf("invalid file part %q in path %q", basename, path)
return
}
err = fmt.Errorf("invalid file part %q in path %q", basename, path)
return
}
+ modtime := node.FileInfo().ModTime()
+ node.Lock()
+ defer node.Unlock()
_, err = node.Child(basename, func(child inode) (inode, error) {
switch child := child.(type) {
case nil:
_, err = node.Child(basename, func(child inode) (inode, error) {
switch child := child.(type) {
case nil:
- child, err = node.FS().newNode(basename, 0755,
node.FileInfo().ModTime()
)
+ child, err = node.FS().newNode(basename, 0755,
modtime
)
if err != nil {
return nil, err
}
if err != nil {
return nil, err
}