- visited = set()
- estimated_size = [0]
- def estimate_collection_cache(obj):
- if obj.get("location", "").startswith("keep:"):
- m = pdh_size.match(obj["location"][5:])
- if m and m.group(1) not in visited:
- visited.add(m.group(1))
- estimated_size[0] += int(m.group(2))
- visit_class(job_order, ("File", "Directory"), estimate_collection_cache)
- logger.info("Estimated size %s", estimated_size)
+ if self.should_estimate_cache_size:
+ visited = set()
+ estimated_size = [0]
+ def estimate_collection_cache(obj):
+ if obj.get("location", "").startswith("keep:"):
+ m = pdh_size.match(obj["location"][5:])
+ if m and m.group(1) not in visited:
+ visited.add(m.group(1))
+ estimated_size[0] += int(m.group(2))
+ visit_class(job_order, ("File", "Directory"), estimate_collection_cache)
+ runtimeContext.collection_cache_size = max(((estimated_size[0]*192) / (1024*1024))+1, 256)
+ self.collection_cache.set_cap(runtimeContext.collection_cache_size*1024*1024)
+
+ logger.info("Using collection cache size %s MiB", runtimeContext.collection_cache_size)