From ad638d5e0e28ab260df272a33bdeaa2b56a96054 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 18 Dec 2014 22:43:23 -0500 Subject: [PATCH] 3198: Refactoring. Added Range object instead of fiddling with arrays, should make the code easier to comprehend. --- sdk/python/arvados/arvfile.py | 26 +++-- sdk/python/arvados/collection.py | 21 ++-- sdk/python/arvados/ranges.py | 60 +++++++--- sdk/python/arvados/stream.py | 71 +++++------ sdk/python/tests/test_collections.py | 169 ++++++++++++++------------- 5 files changed, 193 insertions(+), 154 deletions(-) diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py index 593d4fb873..f544eaafc1 100644 --- a/sdk/python/arvados/arvfile.py +++ b/sdk/python/arvados/arvfile.py @@ -169,7 +169,7 @@ class StreamFileReader(ArvadosFileReaderBase): def _size(self): n = self.segments[-1] - return n[OFFSET] + n[BLOCKSIZE] + return n.range_start + n.range_size @ArvadosFileBase._before_close @retry_method @@ -181,9 +181,10 @@ class StreamFileReader(ArvadosFileReaderBase): data = '' available_chunks = locators_and_ranges(self.segments, self._filepos, size) if available_chunks: - locator, blocksize, segmentoffset, segmentsize = available_chunks[0] - data = self._stream._readfrom(locator+segmentoffset, segmentsize, - num_retries=num_retries) + lr = available_chunks[0] + data = self._stream._readfrom(lr.locator+lr.segment_offset, + lr.segment_size, + num_retries=num_retries) self._filepos += len(data) return data @@ -203,9 +204,9 @@ class StreamFileReader(ArvadosFileReaderBase): def as_manifest(self): manifest_text = ['.'] - manifest_text.extend([d[LOCATOR] for d in self._stream._data_locators]) - manifest_text.extend(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], self.name().replace(' ', '\\040')) for seg in self.segments]) - return CollectionReader(' '.join(manifest_text) + '\n').manifest_text(normalize=True) + manifest_text.extend([d.locator for d in self._stream._data_locators]) + manifest_text.extend(["{}:{}:{}".format(seg.locator, seg.range_size, self.name().replace(' ', '\\040')) for seg in self.segments]) + return manifest_text #arvados.CollectionReader(' '.join(manifest_text) + '\n').manifest_text(normalize=True) class ArvadosFile(ArvadosFileReaderBase): @@ -225,7 +226,7 @@ class ArvadosFile(ArvadosFileReaderBase): fileoffset = 0L for seg in segs: - for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._stream._data_locators, seg[LOCATOR]+seg[OFFSET], seg[SEGMENTSIZE]): + for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._stream._data_locators, seg.locator+seg.range_start, seg[SEGMENTSIZE]): newstream.append([locator, blocksize, streamoffset]) self.segments.append([streamoffset+segmentoffset, segmentsize, fileoffset]) streamoffset += blocksize @@ -259,6 +260,9 @@ class ArvadosFile(ArvadosFileReaderBase): pass def add_segment(self, blocks, pos, size): - for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(blocks, pos, size): - last = self.segments[-1] if self.segments else [0, 0, 0] - self.segments.append([locator, segmentsize, last[OFFSET]+last[BLOCKSIZE], segmentoffset]) + for lr in locators_and_ranges(blocks, pos, size): + last = self.segments[-1] if self.segments else Range(0, 0, 0) + r = Range(lr.locator, last.range_start+last.range_size, lr.segment_size) + r.block_size = lr.block_size + r.segment_offset = lr.segment_offset + self.segments.append(r) diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py index 077115ea8b..b5a9ef99d5 100644 --- a/sdk/python/arvados/collection.py +++ b/sdk/python/arvados/collection.py @@ -9,6 +9,7 @@ from stat import * from .arvfile import ArvadosFileBase, split, ArvadosFile from keep import * from .stream import StreamReader, normalize_stream +from .ranges import Range import config import errors import util @@ -181,7 +182,7 @@ class CollectionReader(CollectionBase): if filename not in streams[streamname]: streams[streamname][filename] = [] for r in f.segments: - streams[streamname][filename].extend(s.locators_and_ranges(r[0], r[1])) + streams[streamname][filename].extend(s.locators_and_ranges(r.locator, r.range_size)) self._streams = [normalize_stream(s, streams[s]) for s in sorted(streams)] @@ -690,10 +691,10 @@ def import_manifest(manifest_text): continue if state == BLOCKS: - s = re.match(r'[0-9a-f]{32}\+(\d)+(\+\S+)*', tok) + s = re.match(r'[0-9a-f]{32}\+(\d+)(\+\S+)*', tok) if s: blocksize = long(s.group(1)) - blocks.append([tok, blocksize, streamoffset]) + blocks.append(Range(tok, streamoffset, blocksize)) streamoffset += blocksize else: state = SEGMENTS @@ -718,12 +719,18 @@ def import_manifest(manifest_text): def export_manifest(item, stream_name="."): buf = "" - print item if isinstance(item, Collection): - for i, j in item.items.values(): - buf += export_manifest(j, stream_name) + stream = {} + for k,v in item.items.items(): + if isinstance(item, Collection): + buf += export_manifest(v, stream_name) + else: + if isinstance(item, ArvadosFile): + buf += str(item.segments) + #stream[k] = [[s.locator, s[4], s[], s[]] for s in item.segments] else: buf += stream_name buf += " " - buf += item.segments + buf += str(item.segments) + buf += "\n" return buf diff --git a/sdk/python/arvados/ranges.py b/sdk/python/arvados/ranges.py index dbd5e3d711..acc1c50136 100644 --- a/sdk/python/arvados/ranges.py +++ b/sdk/python/arvados/ranges.py @@ -1,7 +1,11 @@ -LOCATOR = 0 -BLOCKSIZE = 1 -OFFSET = 2 -SEGMENTSIZE = 3 +class Range(object): + def __init__(self, locator, range_start, range_size): + self.locator = locator + self.range_start = range_start + self.range_size = range_size + + def __repr__(self): + return "[\"%s\", %i, %i]" % (self.locator, self.range_size, self.range_start) def first_block(data_locators, range_start, range_size, debug=False): block_start = 0L @@ -12,8 +16,8 @@ def first_block(data_locators, range_start, range_size, debug=False): hi = len(data_locators) lo = 0 i = int((hi + lo) / 2) - block_size = data_locators[i][BLOCKSIZE] - block_start = data_locators[i][OFFSET] + block_size = data_locators[i].range_size + block_start = data_locators[i].range_start block_end = block_start + block_size if debug: print '---' @@ -30,19 +34,35 @@ def first_block(data_locators, range_start, range_size, debug=False): hi = i i = int((hi + lo) / 2) if debug: print lo, i, hi - block_size = data_locators[i][BLOCKSIZE] - block_start = data_locators[i][OFFSET] + block_size = data_locators[i].range_size + block_start = data_locators[i].range_start block_end = block_start + block_size return i +class LocatorAndRange(object): + def __init__(self, locator, block_size, segment_offset, segment_size): + self.locator = locator + self.block_size = block_size + self.segment_offset = segment_offset + self.segment_size = segment_size + + def __eq__(self, other): + return (self.locator == other.locator and + self.block_size == other.block_size and + self.segment_offset == other.segment_offset and + self.segment_size == other.segment_size) + + def __repr__(self): + return "[\"%s\", %i, %i, %i]" % (self.locator, self.block_size, self.segment_offset, self.segment_size) + def locators_and_ranges(data_locators, range_start, range_size, debug=False): ''' Get blocks that are covered by the range - data_locators: list of [locator, block_size, block_start], assumes that blocks are in order and contigous + data_locators: list of Range objects, assumes that blocks are in order and contigous range_start: start of range range_size: size of range - returns list of [block locator, blocksize, segment offset, segment size] that satisfies the range + returns list of LocatorAndRange objects ''' if range_size == 0: return [] @@ -56,10 +76,12 @@ def locators_and_ranges(data_locators, range_start, range_size, debug=False): return [] while i < len(data_locators): - locator, block_size, block_start = data_locators[i] + dl = data_locators[i] + block_start = dl.range_start + block_size = dl.range_size block_end = block_start + block_size if debug: - print locator, "range_start", range_start, "block_start", block_start, "range_end", range_end, "block_end", block_end + print dl.locator, "range_start", range_start, "block_start", block_start, "range_end", range_end, "block_end", block_end if range_end <= block_start: # range ends before this block starts, so don't look at any more locators break @@ -71,16 +93,16 @@ def locators_and_ranges(data_locators, range_start, range_size, debug=False): if range_start >= block_start and range_end <= block_end: # range starts and ends in this block - resp.append([locator, block_size, range_start - block_start, range_size]) + resp.append(LocatorAndRange(dl.locator, block_size, range_start - block_start, range_size)) elif range_start >= block_start and range_end > block_end: # range starts in this block - resp.append([locator, block_size, range_start - block_start, block_end - range_start]) + resp.append(LocatorAndRange(dl.locator, block_size, range_start - block_start, block_end - range_start)) elif range_start < block_start and range_end > block_end: # range starts in a previous block and extends to further blocks - resp.append([locator, block_size, 0L, block_size]) + resp.append(LocatorAndRange(dl.locator, block_size, 0L, block_size)) elif range_start < block_start and range_end <= block_end: # range starts in a previous block and ends in this block - resp.append([locator, block_size, 0L, range_end - block_start]) + resp.append(LocatorAndRange(dl.locator, block_size, 0L, range_end - block_start)) block_start = block_end i += 1 return resp @@ -88,7 +110,7 @@ def locators_and_ranges(data_locators, range_start, range_size, debug=False): def replace_range(data_locators, range_start, range_size, new_locator, debug=False): ''' Replace a file segment range with a new segment. - data_locators: list of [locator, segment_size, segment_start], assumes that segments are in order and contigous + data_locators: list of Range objects, assumes that segments are in order and contigous range_start: start of range range_size: size of range new_locator: locator for new segment to be inserted @@ -102,9 +124,9 @@ def replace_range(data_locators, range_start, range_size, new_locator, debug=Fal range_end = range_start + range_size last = data_locators[-1] - if (last[OFFSET]+last[BLOCKSIZE]) == range_start: + if (last.range_start+last.range_size) == range_start: # extend last segment - last[BLOCKSIZE] += range_size + last.range_size += range_size return i = first_block(data_locators, range_start, range_size, debug) diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py index ecccefa8c4..d28efcfa8d 100644 --- a/sdk/python/arvados/stream.py +++ b/sdk/python/arvados/stream.py @@ -16,7 +16,8 @@ import errors def normalize_stream(s, stream): ''' s is the stream name - stream is a StreamReader object + stream is a dict mapping each filename to a list in the form [block locator, block size, segment offset (from beginning of block), segment size] + returns the stream as a list of tokens ''' stream_tokens = [s] sortedfiles = list(stream.keys()) @@ -24,29 +25,33 @@ def normalize_stream(s, stream): blocks = {} streamoffset = 0L + # Go through each file and add each referenced block exactly once. for f in sortedfiles: for b in stream[f]: - if b[arvados.LOCATOR] not in blocks: - stream_tokens.append(b[arvados.LOCATOR]) - blocks[b[arvados.LOCATOR]] = streamoffset - streamoffset += b[arvados.BLOCKSIZE] + if b.locator not in blocks: + stream_tokens.append(b.locator) + blocks[b.locator] = streamoffset + streamoffset += b.block_size + # Add the empty block if the stream is otherwise empty. if len(stream_tokens) == 1: stream_tokens.append(config.EMPTY_BLOCK_LOCATOR) for f in sortedfiles: + # Add in file segments current_span = None fout = f.replace(' ', '\\040') for segment in stream[f]: - segmentoffset = blocks[segment[arvados.LOCATOR]] + segment[arvados.OFFSET] + # Collapse adjacent segments + streamoffset = blocks[segment.locator] + segment.segment_offset if current_span is None: - current_span = [segmentoffset, segmentoffset + segment[arvados.SEGMENTSIZE]] + current_span = [streamoffset, streamoffset + segment.segment_size] else: - if segmentoffset == current_span[1]: - current_span[1] += segment[arvados.SEGMENTSIZE] + if streamoffset == current_span[1]: + current_span[1] += segment.segment_size else: stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout)) - current_span = [segmentoffset, segmentoffset + segment[arvados.SEGMENTSIZE]] + current_span = [streamoffset, streamoffset + segment.segment_size] if current_span is not None: stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout)) @@ -78,7 +83,7 @@ class StreamReader(object): s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok) if s: blocksize = long(s.group(1)) - self._data_locators.append([tok, blocksize, streamoffset]) + self._data_locators.append(Range(tok, streamoffset, blocksize)) streamoffset += blocksize continue @@ -88,10 +93,10 @@ class StreamReader(object): size = long(s.group(2)) name = s.group(3).replace('\\040', ' ') if name not in self._files: - self._files[name] = StreamFileReader(self, [[pos, size, 0]], name) + self._files[name] = StreamFileReader(self, [Range(pos, 0, size)], name) else: filereader = self._files[name] - filereader.segments.append([pos, size, filereader.size()]) + filereader.segments.append(Range(pos, filereader.size(), size)) continue raise errors.SyntaxError("Invalid manifest format") @@ -107,7 +112,7 @@ class StreamReader(object): def _size(self): n = self._data_locators[-1] - return n[OFFSET] + n[BLOCKSIZE] + return n.range_start + n.range_size def size(self): return self._size() @@ -131,19 +136,19 @@ class StreamReader(object): if self._keep is None: self._keep = KeepClient(num_retries=self.num_retries) data = [] - for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._data_locators, start, size): - data.append(self._keepget(locator, num_retries=num_retries)[segmentoffset:segmentoffset+segmentsize]) + for lr in locators_and_ranges(self._data_locators, start, size): + data.append(self._keepget(lr.locator, num_retries=num_retries)[lr.segment_offset:lr.segment_offset+lr.segment_size]) return ''.join(data) def manifest_text(self, strip=False): manifest_text = [self.name().replace(' ', '\\040')] if strip: for d in self._data_locators: - m = re.match(r'^[0-9a-f]{32}\+\d+', d[LOCATOR]) + m = re.match(r'^[0-9a-f]{32}\+\d+', d.locator) manifest_text.append(m.group(0)) else: - manifest_text.extend([d[LOCATOR] for d in self._data_locators]) - manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name.replace(' ', '\\040')) + manifest_text.extend([d.locator for d in self._data_locators]) + manifest_text.extend([' '.join(["{}:{}:{}".format(seg.locator, seg.range_size, f.name.replace(' ', '\\040')) for seg in f.segments]) for f in self._files.values()]) return ' '.join(manifest_text) + '\n' @@ -204,8 +209,8 @@ class BufferBlock(object): # def _init_bufferblock(self): # last = self._data_locators[-1] -# streamoffset = last[OFFSET] + last[BLOCKSIZE] -# if last[BLOCKSIZE] == 0: +# streamoffset = last.range_start + last.range_size +# if last.range_size == 0: # del self._data_locators[-1] # self.current_bblock = BufferBlock("bufferblock%i" % len(self.bufferblocks), streamoffset) # self.bufferblocks[self.current_bblock.locator] = self.current_bblock @@ -225,34 +230,34 @@ class BufferBlock(object): # while i < len(tmp_segs): # # Go through each segment and identify segments that include the buffer block # s = tmp_segs[i] -# if s[LOCATOR] < self.current_bblock.locator_list_entry[OFFSET] and (s[LOCATOR] + s[BLOCKSIZE]) > self.current_bblock.locator_list_entry[OFFSET]: +# if s[LOCATOR] < self.current_bblock.locator_list_entry.range_start and (s[LOCATOR] + s.range_size) > self.current_bblock.locator_list_entry.range_start: # # The segment straddles the previous block and the current buffer block. Split the segment. -# b1 = self.current_bblock.locator_list_entry[OFFSET] - s[LOCATOR] -# b2 = (s[LOCATOR] + s[BLOCKSIZE]) - self.current_bblock.locator_list_entry[OFFSET] -# bb_seg = [self.current_bblock.locator_list_entry[OFFSET], b2, s[OFFSET]+b1] -# tmp_segs[i] = [s[LOCATOR], b1, s[OFFSET]] +# b1 = self.current_bblock.locator_list_entry.range_start - s[LOCATOR] +# b2 = (s[LOCATOR] + s.range_size) - self.current_bblock.locator_list_entry.range_start +# bb_seg = [self.current_bblock.locator_list_entry.range_start, b2, s.range_start+b1] +# tmp_segs[i] = [s[LOCATOR], b1, s.range_start] # tmp_segs.insert(i+1, bb_seg) # bufferblock_segs.append(bb_seg) # i += 1 -# elif s[LOCATOR] >= self.current_bblock.locator_list_entry[OFFSET]: +# elif s[LOCATOR] >= self.current_bblock.locator_list_entry.range_start: # # The segment's data is in the buffer block. # bufferblock_segs.append(s) # i += 1 # # Now sum up the segments to get the total bytes # # of the file referencing into the buffer block. -# write_total = sum([s[BLOCKSIZE] for s in bufferblock_segs]) +# write_total = sum([s.range_size for s in bufferblock_segs]) -# if write_total < self.current_bblock.locator_list_entry[BLOCKSIZE]: +# if write_total < self.current_bblock.locator_list_entry.range_size: # # There is more data in the buffer block than is actually accounted for by segments, so # # re-pack into a new buffer by copying over to a new buffer block. # new_bb = BufferBlock(self.current_bblock.locator, -# self.current_bblock.locator_list_entry[OFFSET], +# self.current_bblock.locator_list_entry.range_start, # starting_size=write_total) # for t in bufferblock_segs: -# t_start = t[LOCATOR] - self.current_bblock.locator_list_entry[OFFSET] -# t_end = t_start + t[BLOCKSIZE] -# t[0] = self.current_bblock.locator_list_entry[OFFSET] + new_bb.write_pointer +# t_start = t[LOCATOR] - self.current_bblock.locator_list_entry.range_start +# t_end = t_start + t.range_size +# t[0] = self.current_bblock.locator_list_entry.range_start + new_bb.write_pointer # new_bb.append(self.current_bblock.buffer_block[t_start:t_end]) # self.current_bblock = new_bb diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py index 9882db9503..5ed8716d89 100644 --- a/sdk/python/tests/test_collections.py +++ b/sdk/python/tests/test_collections.py @@ -14,6 +14,7 @@ import unittest import run_test_server import arvados_testutil as tutil +from arvados.ranges import Range, LocatorAndRange class TestResumableWriter(arvados.ResumableCollectionWriter): KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K. @@ -210,97 +211,97 @@ class ArvadosCollectionsTest(run_test_server.TestCaseWithServers, self.assertEqual(arvados.CollectionReader(m8, self.api_client).manifest_text(normalize=True), m8) def test_locators_and_ranges(self): - blocks2 = [['a', 10, 0], - ['b', 10, 10], - ['c', 10, 20], - ['d', 10, 30], - ['e', 10, 40], - ['f', 10, 50]] - - self.assertEqual(arvados.locators_and_ranges(blocks2, 2, 2), [['a', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [['b', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [['c', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [['d', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [['e', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [['f', 10, 2, 2]]) + blocks2 = [Range('a', 0, 10), + Range('b', 10, 10), + Range('c', 20, 10), + Range('d', 30, 10), + Range('e', 40, 10), + Range('f', 50, 10)] + + self.assertEqual(arvados.locators_and_ranges(blocks2, 2, 2), [LocatorAndRange('a', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [LocatorAndRange('b', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [LocatorAndRange('c', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [LocatorAndRange('d', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [LocatorAndRange('e', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [LocatorAndRange('f', 10, 2, 2)]) self.assertEqual(arvados.locators_and_ranges(blocks2, 62, 2), []) self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), []) - self.assertEqual(arvados.locators_and_ranges(blocks2, 0, 2), [['a', 10, 0, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 10, 2), [['b', 10, 0, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 20, 2), [['c', 10, 0, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 30, 2), [['d', 10, 0, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 40, 2), [['e', 10, 0, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 50, 2), [['f', 10, 0, 2]]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 0, 2), [LocatorAndRange('a', 10, 0, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 10, 2), [LocatorAndRange('b', 10, 0, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 20, 2), [LocatorAndRange('c', 10, 0, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 30, 2), [LocatorAndRange('d', 10, 0, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 40, 2), [LocatorAndRange('e', 10, 0, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 50, 2), [LocatorAndRange('f', 10, 0, 2)]) self.assertEqual(arvados.locators_and_ranges(blocks2, 60, 2), []) self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), []) - self.assertEqual(arvados.locators_and_ranges(blocks2, 9, 2), [['a', 10, 9, 1], ['b', 10, 0, 1]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 19, 2), [['b', 10, 9, 1], ['c', 10, 0, 1]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 29, 2), [['c', 10, 9, 1], ['d', 10, 0, 1]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 39, 2), [['d', 10, 9, 1], ['e', 10, 0, 1]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [['e', 10, 9, 1], ['f', 10, 0, 1]]) - self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [['f', 10, 9, 1]]) - - - blocks3 = [['a', 10, 0], - ['b', 10, 10], - ['c', 10, 20], - ['d', 10, 30], - ['e', 10, 40], - ['f', 10, 50], - ['g', 10, 60]] - - self.assertEqual(arvados.locators_and_ranges(blocks3, 2, 2), [['a', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [['b', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [['c', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [['d', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [['e', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [['f', 10, 2, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [['g', 10, 2, 2]]) - - - blocks = [['a', 10, 0], - ['b', 15, 10], - ['c', 5, 25]] + self.assertEqual(arvados.locators_and_ranges(blocks2, 9, 2), [LocatorAndRange('a', 10, 9, 1), LocatorAndRange('b', 10, 0, 1)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 19, 2), [LocatorAndRange('b', 10, 9, 1), LocatorAndRange('c', 10, 0, 1)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 29, 2), [LocatorAndRange('c', 10, 9, 1), LocatorAndRange('d', 10, 0, 1)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 39, 2), [LocatorAndRange('d', 10, 9, 1), LocatorAndRange('e', 10, 0, 1)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [LocatorAndRange('e', 10, 9, 1), LocatorAndRange('f', 10, 0, 1)]) + self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [LocatorAndRange('f', 10, 9, 1)]) + + + blocks3 = [Range('a', 0, 10), + Range('b', 10, 10), + Range('c', 20, 10), + Range('d', 30, 10), + Range('e', 40, 10), + Range('f', 50, 10), + Range('g', 60, 10)] + + self.assertEqual(arvados.locators_and_ranges(blocks3, 2, 2), [LocatorAndRange('a', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [LocatorAndRange('b', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [LocatorAndRange('c', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [LocatorAndRange('d', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [LocatorAndRange('e', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [LocatorAndRange('f', 10, 2, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [LocatorAndRange('g', 10, 2, 2)]) + + + blocks = [Range('a', 0, 10), + Range('b', 10, 15), + Range('c', 25, 5)] self.assertEqual(arvados.locators_and_ranges(blocks, 1, 0), []) - self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [['a', 10, 0, 5]]) - self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [['a', 10, 3, 5]]) - self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [['a', 10, 0, 10]]) - - self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [['a', 10, 0, 10], - ['b', 15, 0, 1]]) - self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [['a', 10, 1, 9], - ['b', 15, 0, 2]]) - self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [['a', 10, 0, 10], - ['b', 15, 0, 15]]) - - self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [['a', 10, 0, 10], - ['b', 15, 0, 15], - ['c', 5, 0, 5]]) - self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [['a', 10, 1, 9], - ['b', 15, 0, 15], - ['c', 5, 0, 5]]) - self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [['a', 10, 0, 10], - ['b', 15, 0, 15], - ['c', 5, 0, 5]]) - - self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [['b', 15, 5, 5]]) - - self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [['a', 10, 8, 2], - ['b', 15, 0, 15]]) - - self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [['a', 10, 8, 2], - ['b', 15, 0, 15], - ['c', 5, 0, 3]]) - - self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [['c', 5, 1, 2]]) - - self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [['a', 10, 9, 1], - ['b', 15, 0, 14]]) - self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [['b', 15, 0, 15]]) - self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [['b', 15, 1, 14], - ['c', 5, 0, 1]]) + self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [LocatorAndRange('a', 10, 0, 5)]) + self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [LocatorAndRange('a', 10, 3, 5)]) + self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [LocatorAndRange('a', 10, 0, 10)]) + + self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [LocatorAndRange('a', 10, 0, 10), + LocatorAndRange('b', 15, 0, 1)]) + self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [LocatorAndRange('a', 10, 1, 9), + LocatorAndRange('b', 15, 0, 2)]) + self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [LocatorAndRange('a', 10, 0, 10), + LocatorAndRange('b', 15, 0, 15)]) + + self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [LocatorAndRange('a', 10, 0, 10), + LocatorAndRange('b', 15, 0, 15), + LocatorAndRange('c', 5, 0, 5)]) + self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [LocatorAndRange('a', 10, 1, 9), + LocatorAndRange('b', 15, 0, 15), + LocatorAndRange('c', 5, 0, 5)]) + self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [LocatorAndRange('a', 10, 0, 10), + LocatorAndRange('b', 15, 0, 15), + LocatorAndRange('c', 5, 0, 5)]) + + self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [LocatorAndRange('b', 15, 5, 5)]) + + self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [LocatorAndRange('a', 10, 8, 2), + LocatorAndRange('b', 15, 0, 15)]) + + self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [LocatorAndRange('a', 10, 8, 2), + LocatorAndRange('b', 15, 0, 15), + LocatorAndRange('c', 5, 0, 3)]) + + self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [LocatorAndRange('c', 5, 1, 2)]) + + self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [LocatorAndRange('a', 10, 9, 1), + LocatorAndRange('b', 15, 0, 14)]) + self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [LocatorAndRange('b', 15, 0, 15)]) + self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [LocatorAndRange('b', 15, 1, 14), + LocatorAndRange('c', 5, 0, 1)]) class MockKeep(object): def __init__(self, content, num_retries=0): -- 2.30.2