-def create_substreams(depth, base_stream_name, max_filesize, data_loc, args):
- streams = [get_stream(base_stream_name, max_filesize, data_loc, args)]
- if depth == 0:
- logger.info("Finished stream {}".format(base_stream_name))
+def create_substreams(depth, base_stream_name, max_filesize, data_loc, args, current_size=0):
+ current_stream = get_stream(base_stream_name, max_filesize, data_loc, args)
+ current_size += len(current_stream)
+ streams = [current_stream]
+
+ if current_size >= (128 * 1024 * 1024):
+ logger.debug("Maximum manifest size reached -- finishing early at {}".format(base_stream_name))
+ elif depth == 0:
+ logger.debug("Finished stream {}".format(base_stream_name))