From 348ef77de2aac95cdaff388da9602362ae269126 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Tue, 29 Mar 2022 12:58:05 -0400 Subject: [PATCH] 18941: FUSE set number of get threads based on cache size Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/python/arvados/commands/get.py | 4 ++-- services/fuse/arvados_fuse/fusedir.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/python/arvados/commands/get.py b/sdk/python/arvados/commands/get.py index a377c149df..c061c70f0e 100755 --- a/sdk/python/arvados/commands/get.py +++ b/sdk/python/arvados/commands/get.py @@ -98,11 +98,11 @@ When getting a collection manifest, strip its access tokens before writing it. """) -parser.add_argument('--threads', type=int, metavar='N', default=2, +parser.add_argument('--threads', type=int, metavar='N', default=4, help=""" Set the number of download threads to be used. Take into account that using lots of threads will increase the RAM requirements. Default is -to use 2 threads. +to use 4 threads. On high latency installations, using a greater number will improve overall throughput. """) diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py index 7de95a0cb1..bea7ed18c4 100644 --- a/services/fuse/arvados_fuse/fusedir.py +++ b/services/fuse/arvados_fuse/fusedir.py @@ -529,11 +529,14 @@ class CollectionDirectory(CollectionDirectoryBase): if uuid_pattern.match(self.collection_locator): coll_reader = arvados.collection.Collection( self.collection_locator, self.api, self.api.keep, - num_retries=self.num_retries) + num_retries=self.num_retries, + get_threads=(self.api.keep.block_cache.cache_max // 64 * 1024 * 1024) ) else: coll_reader = arvados.collection.CollectionReader( self.collection_locator, self.api, self.api.keep, - num_retries=self.num_retries) + num_retries=self.num_retries, + get_threads=(self.api.keep.block_cache.cache_max // 64 * 1024 * 1024) + ) new_collection_record = coll_reader.api_response() or {} # If the Collection only exists in Keep, there will be no API # response. Fill in the fields we need. -- 2.30.2