From 50f8d8487ad5156058087438b670d7c6f8a8d718 Mon Sep 17 00:00:00 2001 From: Tom Clegg Date: Thu, 29 Nov 2018 15:12:25 -0500 Subject: [PATCH] 14538: Reduce max writeahead to 1. Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- sdk/go/arvados/fs_collection.go | 10 ++++++---- sdk/go/arvados/fs_collection_test.go | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go index bc4e4acfa0..6a9a00aa78 100644 --- a/sdk/go/arvados/fs_collection.go +++ b/sdk/go/arvados/fs_collection.go @@ -19,9 +19,11 @@ import ( "time" ) -var maxBlockSize = 1 << 26 - -var concurrentWriters = 4 +var ( + maxBlockSize = 1 << 26 + concurrentWriters = 4 // max goroutines writing to Keep during sync() + writeAheadBlocks = 1 // max background jobs flushing to Keep before blocking writes +) // A CollectionFileSystem is a FileSystem that can be serialized as a // manifest and stored as a collection. @@ -498,7 +500,7 @@ func (fn *filenode) pruneMemSegments() { // TODO: pack/flush small blocks too, when fragmented if fn.throttle == nil { // TODO: share a throttle with filesystem - fn.throttle = newThrottle(concurrentWriters) + fn.throttle = newThrottle(writeAheadBlocks) } for idx, seg := range fn.segments { seg, ok := seg.(*memSegment) diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go index 73f7efda3d..8567a830c5 100644 --- a/sdk/go/arvados/fs_collection_test.go +++ b/sdk/go/arvados/fs_collection_test.go @@ -1057,7 +1057,7 @@ func (s *CollectionFSSuite) TestFlushFullBlocks(c *check.C) { return } - c.Check(len(currentMemExtents()) <= concurrentWriters+1, check.Equals, true) + c.Check(len(currentMemExtents()) <= writeAheadBlocks+1, check.Equals, true) f.(*filehandle).inode.(*filenode).waitPrune() c.Check(currentMemExtents(), check.HasLen, 1) -- 2.30.2