From: Brett Smith Date: Tue, 20 May 2014 13:40:53 +0000 (-0400) Subject: 2752: Refactor Python SDK test_collections. X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/3da7843d8ca43e1e3ef71b33f37f43c33add7b3e 2752: Refactor Python SDK test_collections. This eliminates duplication in setup and teardown code, and clarifies dependencies between tests a bit. --- diff --git a/sdk/python/test_collections.py b/sdk/python/test_collections.py index 7df620d977..5c31f12023 100644 --- a/sdk/python/test_collections.py +++ b/sdk/python/test_collections.py @@ -9,17 +9,11 @@ import bz2 import sys import subprocess -class KeepLocalStoreTest(unittest.TestCase): +class ArvadosCollectionsTest(unittest.TestCase): def setUp(self): os.environ['KEEP_LOCAL_STORE'] = '/tmp' - def runTest(self): - self.assertEqual(arvados.Keep.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put') - self.assertEqual(arvados.Keep.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get') -class LocalCollectionWriterTest(unittest.TestCase): - def setUp(self): - os.environ['KEEP_LOCAL_STORE'] = '/tmp' - def runTest(self): + def write_foo_bar_baz(self): cw = arvados.CollectionWriter() self.assertEqual(cw.current_stream_name(), '.', 'current_stream_name() should be "." now') @@ -32,16 +26,19 @@ class LocalCollectionWriterTest(unittest.TestCase): cw.start_new_stream('baz') cw.write('baz') cw.set_current_file_name('baz.txt') - hash = cw.finish() - self.assertEqual(hash, + return cw.finish() + + def test_keep_local_store(self): + self.assertEqual(arvados.Keep.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put') + self.assertEqual(arvados.Keep.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get') + + def test_local_collection_writer(self): + self.assertEqual(self.write_foo_bar_baz(), 'd6c3b8e571f1b81ebb150a45ed06c884+114', - "resulting manifest hash was {0}, expecting d6c3b8e571f1b81ebb150a45ed06c884+114".format(hash)) + "wrong locator hash for files foo, bar, baz") -class LocalCollectionReaderTest(unittest.TestCase): - def setUp(self): - os.environ['KEEP_LOCAL_STORE'] = '/tmp' - LocalCollectionWriterTest().runTest() - def runTest(self): + def test_local_collection_reader(self): + self.write_foo_bar_baz() cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle') got = [] for s in cr.all_streams(): @@ -63,34 +60,7 @@ class LocalCollectionReaderTest(unittest.TestCase): '', 'reading zero bytes should have returned empty string') -class LocalCollectionManifestSubsetTest(unittest.TestCase): - def setUp(self): - os.environ['KEEP_LOCAL_STORE'] = '/tmp' - LocalCollectionWriterTest().runTest() - def runTest(self): - self._runTest('d6c3b8e571f1b81ebb150a45ed06c884+114', - [[3, '.', 'bar.txt', 'bar'], - [3, '.', 'foo.txt', 'foo'], - [3, './baz', 'baz.txt', 'baz']]) - self._runTest((". %s %s 0:3:foo.txt 3:3:bar.txt\n" % - (arvados.Keep.put("foo"), - arvados.Keep.put("bar"))), - [[3, '.', 'bar.txt', 'bar'], - [3, '.', 'foo.txt', 'foo']]) - self._runTest((". %s %s 0:2:fo.txt 2:4:obar.txt\n" % - (arvados.Keep.put("foo"), - arvados.Keep.put("bar"))), - [[2, '.', 'fo.txt', 'fo'], - [4, '.', 'obar.txt', 'obar']]) - self._runTest((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" % - (arvados.Keep.put("foo"), - arvados.Keep.put("bar"))), - [[2, '.', 'ar.txt', 'ar'], - [2, '.', 'fo.txt', 'fo'], - [2, '.', 'ob.txt', 'ob'], - [0, '.', 'zero.txt', '']]) - - def _runTest(self, collection, expected): + def _test_subset(self, collection, expected): cr = arvados.CollectionReader(collection) for s in cr.all_streams(): for ex in expected: @@ -101,10 +71,31 @@ class LocalCollectionManifestSubsetTest(unittest.TestCase): ex, 'all_files|as_manifest did not preserve manifest contents: got %s expected %s' % (got, ex)) -class LocalCollectionReadlineTest(unittest.TestCase): - def setUp(self): - os.environ['KEEP_LOCAL_STORE'] = '/tmp' - def _runTest(self, what_in, what_out): + def test_collection_manifest_subset(self): + self.write_foo_bar_baz() + self._test_subset('d6c3b8e571f1b81ebb150a45ed06c884+114', + [[3, '.', 'bar.txt', 'bar'], + [3, '.', 'foo.txt', 'foo'], + [3, './baz', 'baz.txt', 'baz']]) + self._test_subset((". %s %s 0:3:foo.txt 3:3:bar.txt\n" % + (arvados.Keep.put("foo"), + arvados.Keep.put("bar"))), + [[3, '.', 'bar.txt', 'bar'], + [3, '.', 'foo.txt', 'foo']]) + self._test_subset((". %s %s 0:2:fo.txt 2:4:obar.txt\n" % + (arvados.Keep.put("foo"), + arvados.Keep.put("bar"))), + [[2, '.', 'fo.txt', 'fo'], + [4, '.', 'obar.txt', 'obar']]) + self._test_subset((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" % + (arvados.Keep.put("foo"), + arvados.Keep.put("bar"))), + [[2, '.', 'ar.txt', 'ar'], + [2, '.', 'fo.txt', 'fo'], + [2, '.', 'ob.txt', 'ob'], + [0, '.', 'zero.txt', '']]) + + def _test_readline(self, what_in, what_out): cw = arvados.CollectionWriter() cw.start_new_file('test.txt') cw.write(what_in) @@ -116,16 +107,14 @@ class LocalCollectionReadlineTest(unittest.TestCase): self.assertEqual(got, what_out, "readlines did not split lines correctly: %s" % got) - def runTest(self): - self._runTest("\na\nbcd\n\nefg\nz", - ["\n", "a\n", "bcd\n", "\n", "efg\n", "z"]) - self._runTest("ab\ncd\n", - ["ab\n", "cd\n"]) -class LocalCollectionEmptyFileTest(unittest.TestCase): - def setUp(self): - os.environ['KEEP_LOCAL_STORE'] = '/tmp' - def runTest(self): + def test_collection_readline(self): + self._test_readline("\na\nbcd\n\nefg\nz", + ["\n", "a\n", "bcd\n", "\n", "efg\n", "z"]) + self._test_readline("ab\ncd\n", + ["ab\n", "cd\n"]) + + def test_collection_empty_file(self): cw = arvados.CollectionWriter() cw.start_new_file('zero.txt') cw.write('') @@ -149,10 +138,7 @@ class LocalCollectionEmptyFileTest(unittest.TestCase): got_sizes += [f.size()] self.assertEqual(got_sizes, expect_sizes, "got wrong file sizes %s, expected %s" % (got_sizes, expect_sizes)) -class LocalCollectionBZ2DecompressionTest(unittest.TestCase): - def setUp(self): - os.environ['KEEP_LOCAL_STORE'] = '/tmp' - def runTest(self): + def test_collection_bz2_decompression(self): n_lines_in = 2**18 data_in = "abc\n" for x in xrange(0, 18): @@ -173,10 +159,7 @@ class LocalCollectionBZ2DecompressionTest(unittest.TestCase): n_lines_in, "decompression returned %d lines instead of %d" % (got, n_lines_in)) -class LocalCollectionGzipDecompressionTest(unittest.TestCase): - def setUp(self): - os.environ['KEEP_LOCAL_STORE'] = '/tmp' - def runTest(self): + def test_collection_gzip_decompression(self): n_lines_in = 2**18 data_in = "abc\n" for x in xrange(0, 18): @@ -202,8 +185,7 @@ class LocalCollectionGzipDecompressionTest(unittest.TestCase): n_lines_in, "decompression returned %d lines instead of %d" % (got, n_lines_in)) -class NormalizedCollectionTest(unittest.TestCase): - def runTest(self): + def test_normalized_collection(self): m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt""" @@ -250,8 +232,7 @@ class NormalizedCollectionTest(unittest.TestCase): """ self.assertEqual(arvados.CollectionReader(m8).manifest_text(), m8) -class LocatorsAndRangesTest(unittest.TestCase): - def runTest(self): + def test_locators_and_ranges(self): blocks2 = [['a', 10, 0], ['b', 10, 10], ['c', 10, 20], @@ -284,7 +265,7 @@ class LocatorsAndRangesTest(unittest.TestCase): self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [['e', 10, 9, 1], ['f', 10, 0, 1]]) self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [['f', 10, 9, 1]]) - + blocks3 = [['a', 10, 0], ['b', 10, 10], ['c', 10, 20], @@ -309,14 +290,14 @@ class LocatorsAndRangesTest(unittest.TestCase): self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [['a', 10, 0, 5]]) self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [['a', 10, 3, 5]]) self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [['a', 10, 0, 10]]) - + self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [['a', 10, 0, 10], ['b', 15, 0, 1]]) self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [['a', 10, 1, 9], ['b', 15, 0, 2]]) self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [['a', 10, 0, 10], ['b', 15, 0, 15]]) - + self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [['a', 10, 0, 10], ['b', 15, 0, 15], ['c', 5, 0, 5]]) @@ -326,25 +307,24 @@ class LocatorsAndRangesTest(unittest.TestCase): self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [['a', 10, 0, 10], ['b', 15, 0, 15], ['c', 5, 0, 5]]) - + self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [['b', 15, 5, 5]]) - + self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [['a', 10, 8, 2], ['b', 15, 0, 15]]) self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [['a', 10, 8, 2], ['b', 15, 0, 15], ['c', 5, 0, 3]]) - + self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [['c', 5, 1, 2]]) - + self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [['a', 10, 9, 1], - ['b', 15, 0, 14]]) + ['b', 15, 0, 14]]) self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [['b', 15, 0, 15]]) self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [['b', 15, 1, 14], ['c', 5, 0, 1]]) -class FileStreamTest(unittest.TestCase): class MockStreamReader(object): def __init__(self, content): self.content = content @@ -352,13 +332,13 @@ class FileStreamTest(unittest.TestCase): def readfrom(self, start, size): return self.content[start:start+size] - def runTest(self): + def test_file_stream(self): content = 'abcdefghijklmnopqrstuvwxyz0123456789' - msr = FileStreamTest.MockStreamReader(content) + msr = self.MockStreamReader(content) segments = [[0, 10, 0], [10, 15, 10], [25, 5, 25]] - + sfr = arvados.StreamFileReader(msr, segments, "test") self.assertEqual(sfr.name(), "test") @@ -389,7 +369,7 @@ class FileStreamTest(unittest.TestCase): segments = [[26, 10, 0], [0, 15, 10], [15, 5, 25]] - + sfr = arvados.StreamFileReader(msr, segments, "test") self.assertEqual(sfr.size(), 30) @@ -417,8 +397,6 @@ class FileStreamTest(unittest.TestCase): self.assertEqual(sfr.tell(), 30) -class StreamReaderTest(unittest.TestCase): - class MockKeep(object): def __init__(self, content): self.content = content @@ -426,11 +404,11 @@ class StreamReaderTest(unittest.TestCase): def get(self, locator): return self.content[locator] - def runTest(self): - keepblocks = {'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': 'abcdefghij', - 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': 'klmnopqrstuvwxy', + def test_stream_reader(self): + keepblocks = {'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': 'abcdefghij', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': 'klmnopqrstuvwxy', 'cccccccccccccccccccccccccccccccc+5': 'z0123'} - mk = StreamReaderTest.MockKeep(keepblocks) + mk = self.MockKeep(keepblocks) sr = arvados.StreamReader([".", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15", "cccccccccccccccccccccccccccccccc+5", "0:30:foo"], mk) @@ -450,8 +428,7 @@ class StreamReaderTest(unittest.TestCase): self.assertEqual(sr.readfrom(25, 5), content[25:30]) self.assertEqual(sr.readfrom(30, 5), '') -class ExtractFileTest(unittest.TestCase): - def runTest(self): + def test_extract_file(self): m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt . 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt @@ -471,3 +448,7 @@ class ExtractFileTest(unittest.TestCase): ". 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt\n") self.assertEqual(arvados.CollectionReader(m1).all_streams()[0].files()['md9sum.txt'].as_manifest(), ". 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt\n") + + +if __name__ == '__main__': + unittest.main()