Added comments and some refactoring.
[arvados.git] / sdk / python / test_collections.py
1 # usage example:
2 #
3 # ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
4
5 import unittest
6 import arvados
7 import os
8 import bz2
9 import sys
10 import subprocess
11
12 class KeepLocalStoreTest(unittest.TestCase):
13     def setUp(self):
14         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
15     def runTest(self):
16         self.assertEqual(arvados.Keep.put('foo'), 'acbd18db4cc2f85cedef654fccc4a4d8+3', 'wrong md5 hash from Keep.put')
17         self.assertEqual(arvados.Keep.get('acbd18db4cc2f85cedef654fccc4a4d8+3'), 'foo', 'wrong data from Keep.get')
18
19 class LocalCollectionWriterTest(unittest.TestCase):
20     def setUp(self):
21         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
22     def runTest(self):
23         cw = arvados.CollectionWriter()
24         self.assertEqual(cw.current_stream_name(), '.',
25                          'current_stream_name() should be "." now')
26         cw.set_current_file_name('foo.txt')
27         cw.write('foo')
28         self.assertEqual(cw.current_file_name(), 'foo.txt',
29                          'current_file_name() should be foo.txt now')
30         cw.start_new_file('bar.txt')
31         cw.write('bar')
32         cw.start_new_stream('baz')
33         cw.write('baz')
34         cw.set_current_file_name('baz.txt')
35         hash = cw.finish()
36         self.assertEqual(hash,
37                          'd6c3b8e571f1b81ebb150a45ed06c884+114',
38                          "resulting manifest hash was {0}, expecting d6c3b8e571f1b81ebb150a45ed06c884+114".format(hash))
39
40 class LocalCollectionReaderTest(unittest.TestCase):
41     def setUp(self):
42         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
43         LocalCollectionWriterTest().runTest()
44     def runTest(self):
45         cr = arvados.CollectionReader('23ca013983d6239e98931cc779e68426+114')
46         got = []
47         for s in cr.all_streams():
48             for f in s.all_files():
49                 got += [[f.size(), f.stream_name(), f.name(), f.read(2**26)]]
50         expected = [[3, '.', 'bar.txt', 'bar'],
51                     [3, '.', 'foo.txt', 'foo'],
52                     [3, './baz', 'baz.txt', 'baz']]
53         self.assertEqual(got,
54                          expected)
55         stream0 = cr.all_streams()[0]
56         self.assertEqual(stream0.readfrom(0, 0),
57                          '',
58                          'reading zero bytes should have returned empty string')
59         self.assertEqual(stream0.readfrom(0, 2**26),
60                          'foobar',
61                          'reading entire stream failed')
62         self.assertEqual(stream0.readfrom(2**26, 0),
63                          '',
64                          'reading zero bytes should have returned empty string')
65
66 class LocalCollectionManifestSubsetTest(unittest.TestCase):
67     def setUp(self):
68         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
69         LocalCollectionWriterTest().runTest()
70     def runTest(self):
71         self._runTest('23ca013983d6239e98931cc779e68426+114',
72                       [[3, '.',     'bar.txt', 'bar'],
73                        [3, '.',     'foo.txt', 'foo'],
74                        [3, './baz', 'baz.txt', 'baz']])
75         self._runTest((". %s %s 0:3:foo.txt 3:3:bar.txt\n" %
76                        (arvados.Keep.put("foo"),
77                         arvados.Keep.put("bar"))),
78                       [[3, '.', 'bar.txt', 'bar'],
79                        [3, '.', 'foo.txt', 'foo']])
80         self._runTest((". %s %s 0:2:fo.txt 2:4:obar.txt\n" %
81                        (arvados.Keep.put("foo"),
82                         arvados.Keep.put("bar"))),
83                       [[2, '.', 'fo.txt', 'fo'],
84                        [4, '.', 'obar.txt', 'obar']])
85         self._runTest((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" %
86                        (arvados.Keep.put("foo"),
87                         arvados.Keep.put("bar"))),
88                       [[2, '.', 'ar.txt', 'ar'],
89                        [2, '.', 'fo.txt', 'fo'],                       
90                        [2, '.', 'ob.txt', 'ob'],
91                        [0, '.', 'zero.txt', '']])
92
93     def _runTest(self, collection, expected):
94         cr = arvados.CollectionReader(collection)
95         for s in cr.all_streams():
96             for ex in expected:
97                 if ex[0] == s:
98                     f = s.files()[ex[2]]
99                     got = [f.size(), f.stream_name(), f.name(), "".join(f.readall(2**26))]
100                     self.assertEqual(got,
101                                      ex,
102                                      'all_files|as_manifest did not preserve manifest contents: got %s expected %s' % (got, ex))
103
104 class LocalCollectionReadlineTest(unittest.TestCase):
105     def setUp(self):
106         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
107     def _runTest(self, what_in, what_out):
108         cw = arvados.CollectionWriter()
109         cw.start_new_file('test.txt')
110         cw.write(what_in)
111         test1 = cw.finish()
112         cr = arvados.CollectionReader(test1)
113         got = []
114         for x in list(cr.all_files())[0].readlines():
115             got += [x]
116         self.assertEqual(got,
117                          what_out,
118                          "readlines did not split lines correctly: %s" % got)
119     def runTest(self):
120         self._runTest("\na\nbcd\n\nefg\nz",
121                       ["\n", "a\n", "bcd\n", "\n", "efg\n", "z"])
122         self._runTest("ab\ncd\n",
123                       ["ab\n", "cd\n"])
124
125 class LocalCollectionEmptyFileTest(unittest.TestCase):
126     def setUp(self):
127         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
128     def runTest(self):
129         cw = arvados.CollectionWriter()
130         cw.start_new_file('zero.txt')
131         cw.write('')
132
133         print 'stuff'
134
135         self.assertEqual(cw.manifest_text(), ". 0:0:zero.txt\n")
136         self.check_manifest_file_sizes(cw.manifest_text(), [0])
137         cw = arvados.CollectionWriter()
138         cw.start_new_file('zero.txt')
139         cw.write('')
140         cw.start_new_file('one.txt')
141         cw.write('1')
142         cw.start_new_stream('foo')
143         cw.start_new_file('zero.txt')
144         cw.write('')
145         self.check_manifest_file_sizes(cw.manifest_text(), [1,0,0])
146
147     def check_manifest_file_sizes(self, manifest_text, expect_sizes):
148         cr = arvados.CollectionReader(manifest_text)
149         got_sizes = []
150         for f in cr.all_files():
151             got_sizes += [f.size()]
152         self.assertEqual(got_sizes, expect_sizes, "got wrong file sizes %s, expected %s" % (got_sizes, expect_sizes))
153
154 class LocalCollectionBZ2DecompressionTest(unittest.TestCase):
155     def setUp(self):
156         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
157     def runTest(self):
158         n_lines_in = 2**18
159         data_in = "abc\n"
160         for x in xrange(0, 18):
161             data_in += data_in
162         compressed_data_in = bz2.compress(data_in)
163         cw = arvados.CollectionWriter()
164         cw.start_new_file('test.bz2')
165         cw.write(compressed_data_in)
166         bz2_manifest = cw.manifest_text()
167
168         cr = arvados.CollectionReader(bz2_manifest)
169         got = 0
170         for x in list(cr.all_files())[0].readlines():
171             self.assertEqual(x, "abc\n", "decompression returned wrong data: %s" % x)
172             got += 1
173         self.assertEqual(got,
174                          n_lines_in,
175                          "decompression returned %d lines instead of %d" % (got, n_lines_in))
176
177 class LocalCollectionGzipDecompressionTest(unittest.TestCase):
178     def setUp(self):
179         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
180     def runTest(self):
181         n_lines_in = 2**18
182         data_in = "abc\n"
183         for x in xrange(0, 18):
184             data_in += data_in
185         p = subprocess.Popen(["gzip", "-1cn"],
186                              stdout=subprocess.PIPE,
187                              stdin=subprocess.PIPE,
188                              stderr=subprocess.PIPE,
189                              shell=False, close_fds=True)
190         compressed_data_in, stderrdata = p.communicate(data_in)
191
192         cw = arvados.CollectionWriter()
193         cw.start_new_file('test.gz')
194         cw.write(compressed_data_in)
195         gzip_manifest = cw.manifest_text()
196
197         cr = arvados.CollectionReader(gzip_manifest)
198         got = 0
199         for x in list(cr.all_files())[0].readlines():
200             self.assertEqual(x, "abc\n", "decompression returned wrong data: %s" % x)
201             got += 1
202         self.assertEqual(got,
203                          n_lines_in,
204                          "decompression returned %d lines instead of %d" % (got, n_lines_in))
205
206 class NormalizedCollectionTest(unittest.TestCase):
207     def runTest(self):
208         m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
209 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
210 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt"""
211         self.assertEqual(arvados.CollectionReader(m1).manifest_text(),
212                          """. 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt
213 """)
214
215         m2 = """. 204e43b8a1185621ca55a94839582e6f+67108864 b9677abbac956bd3e86b1deb28dfac03+67108864 fc15aff2a762b13f521baf042140acec+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:227212247:var-GS000016015-ASM.tsv.bz2
216 """
217         self.assertEqual(arvados.CollectionReader(m2).manifest_text(), m2)
218
219         m3 = """. 5348b82a029fd9e971a811ce1f71360b+43 3:40:md5sum.txt
220 . 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
221 . 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt"""
222         self.assertEqual(arvados.CollectionReader(m3).manifest_text(),
223                          """. 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 3:124:md5sum.txt
224 """)
225
226         m4 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
227 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
228 ./foo 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar"""
229         self.assertEqual(arvados.CollectionReader(m4).manifest_text(),
230                          """./foo 204e43b8a1185621ca55a94839582e6f+67108864 323d2a3ce20370c4ca1d3462a344f8fd+25885655 0:3:bar 67108864:3:bar
231 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
232 """)
233
234         m5 = """. 204e43b8a1185621ca55a94839582e6f+67108864 0:3:foo/bar
235 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
236 ./foo 204e43b8a1185621ca55a94839582e6f+67108864 3:3:bar"""
237         self.assertEqual(arvados.CollectionReader(m5).manifest_text(),
238                          """./foo 204e43b8a1185621ca55a94839582e6f+67108864 0:6:bar
239 ./zzz 204e43b8a1185621ca55a94839582e6f+67108864 0:999:zzz
240 """)
241
242         with open('testdata/1000G_ref_manifest') as f6:
243             m6 = f6.read()
244             self.assertEqual(arvados.CollectionReader(m6).manifest_text(), m6)
245
246         with open('testdata/jlake_manifest') as f7:
247             m7 = f7.read()
248             self.assertEqual(arvados.CollectionReader(m7).manifest_text(), m7)
249
250         m8 = """./a\\040b\\040c 59ca0efa9f5633cb0371bbc0355478d8+13 0:13:hello\\040world.txt
251 """
252         self.assertEqual(arvados.CollectionReader(m8).manifest_text(), m8)
253
254 class LocatorsAndRangesTest(unittest.TestCase):
255     def runTest(self):
256         blocks2 = [['a', 10, 0],
257                   ['b', 10, 10],
258                   ['c', 10, 20],
259                   ['d', 10, 30],
260                   ['e', 10, 40],
261                   ['f', 10, 50]]
262
263         self.assertEqual(arvados.locators_and_ranges(blocks2,  2,  2), [['a', 10, 2, 2]])
264         self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [['b', 10, 2, 2]])
265         self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [['c', 10, 2, 2]])
266         self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [['d', 10, 2, 2]])
267         self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [['e', 10, 2, 2]])
268         self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [['f', 10, 2, 2]])
269         self.assertEqual(arvados.locators_and_ranges(blocks2, 62, 2), [])
270         self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
271
272         self.assertEqual(arvados.locators_and_ranges(blocks2,  0,  2), [['a', 10, 0, 2]])
273         self.assertEqual(arvados.locators_and_ranges(blocks2, 10, 2), [['b', 10, 0, 2]])
274         self.assertEqual(arvados.locators_and_ranges(blocks2, 20, 2), [['c', 10, 0, 2]])
275         self.assertEqual(arvados.locators_and_ranges(blocks2, 30, 2), [['d', 10, 0, 2]])
276         self.assertEqual(arvados.locators_and_ranges(blocks2, 40, 2), [['e', 10, 0, 2]])
277         self.assertEqual(arvados.locators_and_ranges(blocks2, 50, 2), [['f', 10, 0, 2]])
278         self.assertEqual(arvados.locators_and_ranges(blocks2, 60, 2), [])
279         self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
280
281         self.assertEqual(arvados.locators_and_ranges(blocks2,  9,  2), [['a', 10, 9, 1], ['b', 10, 0, 1]])
282         self.assertEqual(arvados.locators_and_ranges(blocks2, 19, 2), [['b', 10, 9, 1], ['c', 10, 0, 1]])
283         self.assertEqual(arvados.locators_and_ranges(blocks2, 29, 2), [['c', 10, 9, 1], ['d', 10, 0, 1]])
284         self.assertEqual(arvados.locators_and_ranges(blocks2, 39, 2), [['d', 10, 9, 1], ['e', 10, 0, 1]])
285         self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [['e', 10, 9, 1], ['f', 10, 0, 1]])
286         self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [['f', 10, 9, 1]])
287
288         
289         blocks3 = [['a', 10, 0],
290                   ['b', 10, 10],
291                   ['c', 10, 20],
292                   ['d', 10, 30],
293                   ['e', 10, 40],
294                   ['f', 10, 50],
295                   ['g', 10, 60]]
296
297         self.assertEqual(arvados.locators_and_ranges(blocks3,  2,  2), [['a', 10, 2, 2]])
298         self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [['b', 10, 2, 2]])
299         self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [['c', 10, 2, 2]])
300         self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [['d', 10, 2, 2]])
301         self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [['e', 10, 2, 2]])
302         self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [['f', 10, 2, 2]])
303         self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [['g', 10, 2, 2]])
304
305
306         blocks = [['a', 10, 0],
307                   ['b', 15, 10],
308                   ['c', 5, 25]]
309         self.assertEqual(arvados.locators_and_ranges(blocks, 1, 0), [])
310         self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [['a', 10, 0, 5]])
311         self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [['a', 10, 3, 5]])
312         self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [['a', 10, 0, 10]])
313         
314         self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [['a', 10, 0, 10],
315                                                                       ['b', 15, 0, 1]])
316         self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [['a', 10, 1, 9],
317                                                                       ['b', 15, 0, 2]])
318         self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [['a', 10, 0, 10],
319                                                                       ['b', 15, 0, 15]])
320         
321         self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [['a', 10, 0, 10],
322                                                                       ['b', 15, 0, 15],
323                                                                       ['c', 5, 0, 5]])
324         self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [['a', 10, 1, 9],
325                                                                       ['b', 15, 0, 15],
326                                                                       ['c', 5, 0, 5]])
327         self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [['a', 10, 0, 10],
328                                                                       ['b', 15, 0, 15],
329                                                                       ['c', 5, 0, 5]])
330         
331         self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [['b', 15, 5, 5]])
332         
333         self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [['a', 10, 8, 2],
334                                                                       ['b', 15, 0, 15]])
335
336         self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [['a', 10, 8, 2],
337                                                                       ['b', 15, 0, 15],
338                                                                       ['c', 5, 0, 3]])
339         
340         self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [['c', 5, 1, 2]])
341         
342         self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [['a', 10, 9, 1],
343                                                                       ['b', 15, 0, 14]])        
344         self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [['b', 15, 0, 15]])
345         self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [['b', 15, 1, 14],
346                                                                        ['c', 5, 0, 1]])
347
348 class FileStreamTest(unittest.TestCase):
349     class MockStreamReader(object):
350         def __init__(self, content):
351             self.content = content
352
353         def readfrom(self, start, size):
354             return self.content[start:start+size]
355
356     def runTest(self):
357         content = 'abcdefghijklmnopqrstuvwxyz0123456789'
358         msr = FileStreamTest.MockStreamReader(content)
359         segments = [[0, 10, 0],
360                     [10, 15, 10],
361                     [25, 5, 25]]
362         
363         sfr = arvados.StreamFileReader(msr, segments, "test")
364
365         self.assertEqual(sfr.name(), "test")
366         self.assertEqual(sfr.size(), 30)
367
368         self.assertEqual(sfr.readfrom(0, 30), content[0:30])
369         self.assertEqual(sfr.readfrom(2, 30), content[2:30])
370
371         self.assertEqual(sfr.readfrom(2, 8), content[2:10])
372         self.assertEqual(sfr.readfrom(0, 10), content[0:10])
373
374         self.assertEqual(sfr.tell(), 0)
375         self.assertEqual(sfr.read(5), content[0:5])
376         self.assertEqual(sfr.tell(), 5)
377         self.assertEqual(sfr.read(5), content[5:10])
378         self.assertEqual(sfr.tell(), 10)
379         self.assertEqual(sfr.read(5), content[10:15])
380         self.assertEqual(sfr.tell(), 15)
381         self.assertEqual(sfr.read(5), content[15:20])
382         self.assertEqual(sfr.tell(), 20)
383         self.assertEqual(sfr.read(5), content[20:25])
384         self.assertEqual(sfr.tell(), 25)
385         self.assertEqual(sfr.read(5), content[25:30])
386         self.assertEqual(sfr.tell(), 30)
387         self.assertEqual(sfr.read(5), '')
388         self.assertEqual(sfr.tell(), 30)
389
390         segments = [[26, 10, 0],
391                     [0, 15, 10],
392                     [15, 5, 25]]
393         
394         sfr = arvados.StreamFileReader(msr, segments, "test")
395
396         self.assertEqual(sfr.size(), 30)
397
398         self.assertEqual(sfr.readfrom(0, 30), content[26:36] + content[0:20])
399         self.assertEqual(sfr.readfrom(2, 30), content[28:36] + content[0:20])
400
401         self.assertEqual(sfr.readfrom(2, 8), content[28:36])
402         self.assertEqual(sfr.readfrom(0, 10), content[26:36])
403
404         self.assertEqual(sfr.tell(), 0)
405         self.assertEqual(sfr.read(5), content[26:31])
406         self.assertEqual(sfr.tell(), 5)
407         self.assertEqual(sfr.read(5), content[31:36])
408         self.assertEqual(sfr.tell(), 10)
409         self.assertEqual(sfr.read(5), content[0:5])
410         self.assertEqual(sfr.tell(), 15)
411         self.assertEqual(sfr.read(5), content[5:10])
412         self.assertEqual(sfr.tell(), 20)
413         self.assertEqual(sfr.read(5), content[10:15])
414         self.assertEqual(sfr.tell(), 25)
415         self.assertEqual(sfr.read(5), content[15:20])
416         self.assertEqual(sfr.tell(), 30)
417         self.assertEqual(sfr.read(5), '')
418         self.assertEqual(sfr.tell(), 30)
419
420
421 class StreamReaderTest(unittest.TestCase):
422
423     class MockKeep(object):
424         def __init__(self, content):
425             self.content = content
426
427         def get(self, locator):
428             return self.content[locator]
429
430     def runTest(self):
431         keepblocks = {'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': 'abcdefghij', 
432                       'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': 'klmnopqrstuvwxy', 
433                       'cccccccccccccccccccccccccccccccc+5': 'z0123'}
434         mk = StreamReaderTest.MockKeep(keepblocks)
435
436         sr = arvados.StreamReader([".", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15", "cccccccccccccccccccccccccccccccc+5", "0:30:foo"], mk)
437
438         content = 'abcdefghijklmnopqrstuvwxyz0123456789'
439
440         self.assertEqual(sr.readfrom(0, 30), content[0:30])
441         self.assertEqual(sr.readfrom(2, 30), content[2:30])
442
443         self.assertEqual(sr.readfrom(2, 8), content[2:10])
444         self.assertEqual(sr.readfrom(0, 10), content[0:10])
445
446         self.assertEqual(sr.readfrom(0, 5), content[0:5])
447         self.assertEqual(sr.readfrom(5, 5), content[5:10])
448         self.assertEqual(sr.readfrom(10, 5), content[10:15])
449         self.assertEqual(sr.readfrom(15, 5), content[15:20])
450         self.assertEqual(sr.readfrom(20, 5), content[20:25])
451         self.assertEqual(sr.readfrom(25, 5), content[25:30])
452         self.assertEqual(sr.readfrom(30, 5), '')
453