1 # Copyright (C) The Arvados Authors. All rights reserved.
3 # SPDX-License-Identifier: AGPL-3.0
7 import crunchstat_summary.command
15 TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
18 class ReportDiff(unittest.TestCase):
19 def diff_known_report(self, logfile, cmd):
20 expectfile = logfile+'.report'
21 expect = open(expectfile).readlines()
22 self.diff_report(cmd, expect, expectfile=expectfile)
24 def diff_report(self, cmd, expect, expectfile=None):
25 got = [x+"\n" for x in cmd.report().strip("\n").split("\n")]
26 self.assertEqual(got, expect, "\n"+"".join(difflib.context_diff(
27 expect, got, fromfile=expectfile, tofile="(generated)")))
30 class SummarizeFile(ReportDiff):
31 def test_example_files(self):
32 for fnm in glob.glob(os.path.join(TESTS_DIR, '*.txt.gz')):
33 logfile = os.path.join(TESTS_DIR, fnm)
34 args = crunchstat_summary.command.ArgumentParser().parse_args(
35 ['--log-file', logfile])
36 cmd = crunchstat_summary.command.Command(args)
38 self.diff_known_report(logfile, cmd)
41 class HTMLFromFile(ReportDiff):
42 def test_example_files(self):
43 # Note we don't test the output content at all yet; we're
44 # mainly just verifying the --format=html option isn't ignored
45 # and the HTML code path doesn't crash.
46 for fnm in glob.glob(os.path.join(TESTS_DIR, '*.txt.gz')):
47 logfile = os.path.join(TESTS_DIR, fnm)
48 args = crunchstat_summary.command.ArgumentParser().parse_args(
49 ['--format=html', '--log-file', logfile])
50 cmd = crunchstat_summary.command.Command(args)
52 self.assertRegexpMatches(cmd.report(), r'(?is)<html>.*</html>\s*$')
55 class SummarizeEdgeCases(unittest.TestCase):
56 def test_error_messages(self):
57 logfile = open(os.path.join(TESTS_DIR, 'crunchstat_error_messages.txt'))
58 s = crunchstat_summary.summarizer.Summarizer(logfile)
62 class SummarizeContainer(ReportDiff):
64 'uuid': '9tee4-dz642-mjfb0i5hzojp16a',
65 'log': '9tee4-4zz18-ihyzym9tcwjwg4r',
68 'uuid': '9tee4-xvhdp-uper95jktm10d3w',
70 'container_uuid': fake_container['uuid'],
72 logfile = os.path.join(
73 TESTS_DIR, 'container_9tee4-dz642-mjfb0i5hzojp16a-crunchstat.txt.gz')
75 @mock.patch('arvados.collection.CollectionReader')
76 @mock.patch('arvados.api')
77 def test_container(self, mock_api, mock_cr):
78 mock_api().container_requests().index().execute.return_value = {'items':[]}
79 mock_api().container_requests().get().execute.return_value = self.fake_request
80 mock_api().containers().get().execute.return_value = self.fake_container
81 mock_cr().__iter__.return_value = [
82 'crunch-run.txt', 'stderr.txt', 'node-info.txt',
83 'container.json', 'crunchstat.txt']
84 mock_cr().open.return_value = gzip.open(self.logfile)
85 args = crunchstat_summary.command.ArgumentParser().parse_args(
86 ['--job', self.fake_request['uuid']])
87 cmd = crunchstat_summary.command.Command(args)
89 self.diff_known_report(self.logfile, cmd)
92 class SummarizeJob(ReportDiff):
93 fake_job_uuid = '4xphq-8i9sb-jq0ekny1xou3zoh'
94 fake_log_id = 'fake-log-collection-id'
96 'uuid': fake_job_uuid,
99 logfile = os.path.join(TESTS_DIR, 'logfile_20151204190335.txt.gz')
101 @mock.patch('arvados.collection.CollectionReader')
102 @mock.patch('arvados.api')
103 def test_job_report(self, mock_api, mock_cr):
104 mock_api().jobs().get().execute.return_value = self.fake_job
105 mock_cr().__iter__.return_value = ['fake-logfile.txt']
106 mock_cr().open.return_value = gzip.open(self.logfile)
107 args = crunchstat_summary.command.ArgumentParser().parse_args(
108 ['--job', self.fake_job_uuid])
109 cmd = crunchstat_summary.command.Command(args)
111 self.diff_known_report(self.logfile, cmd)
112 mock_api().jobs().get.assert_called_with(uuid=self.fake_job_uuid)
113 mock_cr.assert_called_with(self.fake_log_id)
114 mock_cr().open.assert_called_with('fake-logfile.txt')
117 class SummarizePipeline(ReportDiff):
119 'uuid': 'zzzzz-d1hrv-i3e77t9z5y8j9cc',
120 'owner_uuid': 'zzzzz-tpzed-xurymjxw79nv3jz',
121 'components': collections.OrderedDict([
124 'uuid': 'zzzzz-8i9sb-000000000000000',
125 'log': 'fake-log-pdh-0',
126 'runtime_constraints': {
127 'min_ram_mb_per_node': 900,
128 'min_cores_per_node': 1,
134 'uuid': 'zzzzz-8i9sb-000000000000001',
135 'log': 'fake-log-pdh-1',
136 'runtime_constraints': {
137 'min_ram_mb_per_node': 900,
138 'min_cores_per_node': 1,
142 ['no-job-assigned', {}],
145 'uuid': 'zzzzz-8i9sb-xxxxxxxxxxxxxxx',
150 'uuid': 'zzzzz-8i9sb-000000000000002',
151 'log': 'fake-log-pdh-2',
152 'runtime_constraints': {
153 'min_ram_mb_per_node': 900,
154 'min_cores_per_node': 1,
160 @mock.patch('arvados.collection.CollectionReader')
161 @mock.patch('arvados.api')
162 def test_pipeline(self, mock_api, mock_cr):
163 logfile = os.path.join(TESTS_DIR, 'logfile_20151204190335.txt.gz')
164 mock_api().pipeline_instances().get().execute. \
165 return_value = self.fake_instance
166 mock_cr().__iter__.return_value = ['fake-logfile.txt']
167 mock_cr().open.side_effect = [gzip.open(logfile) for _ in range(3)]
168 args = crunchstat_summary.command.ArgumentParser().parse_args(
169 ['--pipeline-instance', self.fake_instance['uuid']])
170 cmd = crunchstat_summary.command.Command(args)
174 line for line in open(logfile+'.report').readlines()
175 if not line.startswith('#!! ')]
177 ['### Summary for foo (zzzzz-8i9sb-000000000000000)\n'] +
178 job_report + ['\n'] +
179 ['### Summary for bar (zzzzz-8i9sb-000000000000001)\n'] +
180 job_report + ['\n'] +
181 ['### Summary for unfinished-job (zzzzz-8i9sb-xxxxxxxxxxxxxxx)\n',
182 '(no report generated)\n',
184 ['### Summary for baz (zzzzz-8i9sb-000000000000002)\n'] +
186 self.diff_report(cmd, expect)
187 mock_cr.assert_has_calls(
189 mock.call('fake-log-pdh-0'),
190 mock.call('fake-log-pdh-1'),
191 mock.call('fake-log-pdh-2'),
193 mock_cr().open.assert_called_with('fake-logfile.txt')