tools/crunchstat-summary/tests/test_examples.py

   1 import arvados
   2 import collections
   3 import crunchstat_summary.command
   4 import crunchstat_summary.summarizer
   5 import difflib
   6 import glob
   7 import gzip
   8 import mock
   9 import os
  10 import unittest
  11
  12
  13 TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
  14
  15 class ReportDiff(unittest.TestCase):
  16     def diff_known_report(self, logfile, summarizer):
  17         expectfile = logfile+'.report'
  18         expect = open(expectfile).readlines()
  19         self.diff_report(summarizer, expect, expectfile=expectfile)
  20
  21     def diff_report(self, summarizer, expect, expectfile=None):
  22         got = [x+"\n" for x in summarizer.report().strip("\n").split("\n")]
  23         self.assertEqual(got, expect, "\n"+"".join(difflib.context_diff(
  24             expect, got, fromfile=expectfile, tofile="(generated)")))
  25
  26
  27 class ExampleLogsTestCase(ReportDiff):
  28     def test_example_files(self):
  29         for fnm in glob.glob(os.path.join(TESTS_DIR, '*.txt.gz')):
  30             logfile = os.path.join(TESTS_DIR, fnm)
  31             args = crunchstat_summary.command.ArgumentParser().parse_args(
  32                 ['--log-file', logfile])
  33             summarizer = crunchstat_summary.command.Command(args).summarizer()
  34             summarizer.run()
  35             self.diff_known_report(logfile, summarizer)
  36
  37
  38 class LookupJobUUID(ReportDiff):
  39     fake_uuid = 'zzzzz-8i9sb-jq0ekny1xou3zoh'
  40
  41     @mock.patch('arvados.collection.CollectionReader')
  42     @mock.patch('arvados.api')
  43     def test_job_uuid(self, mock_api, mock_cr):
  44         logfile = os.path.join(TESTS_DIR, 'logfile_20151204190335.txt.gz')
  45         mock_api().jobs().get().execute.return_value = {'log': 'fake-uuid'}
  46         mock_cr().__iter__.return_value = ['fake-logfile.txt']
  47         mock_cr().open.return_value = gzip.open(logfile)
  48         args = crunchstat_summary.command.ArgumentParser().parse_args(
  49             ['--job', self.fake_uuid])
  50         summarizer = crunchstat_summary.command.Command(args).summarizer()
  51         summarizer.run()
  52         self.diff_known_report(logfile, summarizer)
  53         mock_api().jobs().get.assert_called_with(uuid=self.fake_uuid)
  54         mock_cr().open.assert_called_with('fake-logfile.txt')
  55
  56
  57 class SummarizePipeline(ReportDiff):
  58     fake_instance = {
  59         'uuid': 'zzzzz-d1hrv-i3e77t9z5y8j9cc',
  60         'owner_uuid': 'zzzzz-tpzed-xurymjxw79nv3jz',
  61         'components': collections.OrderedDict([
  62             ['foo', {
  63                 'job': {
  64                     'uuid': 'zzzzz-8i9sb-000000000000000',
  65                     'log': 'fake-log-pdh-0',
  66                 },
  67             }],
  68             ['bar', {
  69                 'job': {
  70                     'uuid': 'zzzzz-8i9sb-000000000000001',
  71                     'log': 'fake-log-pdh-1',
  72                 },
  73             }],
  74             ['no-job-assigned', {}],
  75             ['unfinished-job', {
  76                 'job': {
  77                     'uuid': 'zzzzz-8i9sb-xxxxxxxxxxxxxxx',
  78                 },
  79             }],
  80             ['baz', {
  81                 'job': {
  82                     'uuid': 'zzzzz-8i9sb-000000000000002',
  83                     'log': 'fake-log-pdh-2',
  84                 },
  85             }]]),
  86     }
  87
  88     @mock.patch('arvados.collection.CollectionReader')
  89     @mock.patch('arvados.api')
  90     def test_pipeline(self, mock_api, mock_cr):
  91         logfile = os.path.join(TESTS_DIR, 'logfile_20151204190335.txt.gz')
  92         mock_api().pipeline_instances().get().execute. \
  93             return_value = self.fake_instance
  94         mock_cr().__iter__.return_value = ['fake-logfile.txt']
  95         mock_cr().open.side_effect = [gzip.open(logfile) for _ in range(3)]
  96         args = crunchstat_summary.command.ArgumentParser().parse_args(
  97             ['--pipeline-instance', self.fake_instance['uuid']])
  98         summarizer = crunchstat_summary.command.Command(args).summarizer()
  99         summarizer.run()
 100
 101         expect = (
 102             ['### Summary for foo (zzzzz-8i9sb-000000000000000)\n'] +
 103             open(logfile+'.report').readlines() + ['\n'] +
 104             ['### Summary for bar (zzzzz-8i9sb-000000000000001)\n'] +
 105             open(logfile+'.report').readlines() + ['\n'] +
 106             ['### Summary for baz (zzzzz-8i9sb-000000000000002)\n'] +
 107             open(logfile+'.report').readlines())
 108         self.diff_report(summarizer, expect)
 109         mock_cr.assert_has_calls(
 110             [
 111                 mock.call('fake-log-pdh-0'),
 112                 mock.call('fake-log-pdh-1'),
 113                 mock.call('fake-log-pdh-2'),
 114             ], any_order=True)
 115         mock_cr().open.assert_called_with('fake-logfile.txt')