import difflib
import glob
import gzip
+from io import open
import mock
import os
import unittest
+from crunchstat_summary.command import UTF8Decode
+
TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
class ReportDiff(unittest.TestCase):
def diff_known_report(self, logfile, cmd):
expectfile = logfile+'.report'
- expect = open(expectfile).readlines()
+ with open(expectfile, encoding='utf-8') as f:
+ expect = f.readlines()
self.diff_report(cmd, expect, expectfile=expectfile)
- def diff_report(self, cmd, expect, expectfile=None):
+ def diff_report(self, cmd, expect, expectfile='(expected)'):
got = [x+"\n" for x in cmd.report().strip("\n").split("\n")]
self.assertEqual(got, expect, "\n"+"".join(difflib.context_diff(
expect, got, fromfile=expectfile, tofile="(generated)")))
cmd.run()
self.assertRegexpMatches(cmd.report(), r'(?is)<html>.*</html>\s*$')
-
class SummarizeEdgeCases(unittest.TestCase):
def test_error_messages(self):
- logfile = open(os.path.join(TESTS_DIR, 'crunchstat_error_messages.txt'))
+ logfile = open(os.path.join(TESTS_DIR, 'crunchstat_error_messages.txt'), encoding='utf-8')
s = crunchstat_summary.summarizer.Summarizer(logfile)
s.run()
+class SummarizeContainer(ReportDiff):
+ fake_container = {
+ 'uuid': '9tee4-dz642-lymtndkpy39eibk',
+ 'created_at': '2017-08-18T14:27:25.371388141',
+ 'log': '9tee4-4zz18-ihyzym9tcwjwg4r',
+ }
+ fake_request = {
+ 'uuid': '9tee4-xvhdp-uper95jktm10d3w',
+ 'name': 'container',
+ 'created_at': '2017-08-18T14:27:25.242339223Z',
+ 'container_uuid': fake_container['uuid'],
+ }
+ reportfile = os.path.join(
+ TESTS_DIR, 'container_9tee4-dz642-lymtndkpy39eibk.txt.gz')
+ logfile = os.path.join(
+ TESTS_DIR, 'container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz')
+ arvmountlog = os.path.join(
+ TESTS_DIR, 'container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz')
+
+ @mock.patch('arvados.collection.CollectionReader')
+ @mock.patch('arvados.api')
+ def test_container(self, mock_api, mock_cr):
+ mock_api().container_requests().index().execute.return_value = {'items':[]}
+ mock_api().container_requests().get().execute.return_value = self.fake_request
+ mock_api().containers().get().execute.return_value = self.fake_container
+ mock_cr().__iter__.return_value = [
+ 'crunch-run.txt', 'stderr.txt', 'node-info.txt',
+ 'container.json', 'crunchstat.txt', 'arv-mount.txt']
+ def _open(n):
+ if n == "crunchstat.txt":
+ return UTF8Decode(gzip.open(self.logfile))
+ elif n == "arv-mount.txt":
+ return UTF8Decode(gzip.open(self.arvmountlog))
+ mock_cr().open.side_effect = _open
+ args = crunchstat_summary.command.ArgumentParser().parse_args(
+ ['--job', self.fake_request['uuid']])
+ cmd = crunchstat_summary.command.Command(args)
+ cmd.run()
+ self.diff_known_report(self.reportfile, cmd)
+
+
class SummarizeJob(ReportDiff):
fake_job_uuid = '4xphq-8i9sb-jq0ekny1xou3zoh'
fake_log_id = 'fake-log-collection-id'
def test_job_report(self, mock_api, mock_cr):
mock_api().jobs().get().execute.return_value = self.fake_job
mock_cr().__iter__.return_value = ['fake-logfile.txt']
- mock_cr().open.return_value = gzip.open(self.logfile)
+ mock_cr().open.return_value = UTF8Decode(gzip.open(self.logfile))
args = crunchstat_summary.command.ArgumentParser().parse_args(
['--job', self.fake_job_uuid])
cmd = crunchstat_summary.command.Command(args)
mock_api().pipeline_instances().get().execute. \
return_value = self.fake_instance
mock_cr().__iter__.return_value = ['fake-logfile.txt']
- mock_cr().open.side_effect = [gzip.open(logfile) for _ in range(3)]
+ mock_cr().open.side_effect = [UTF8Decode(gzip.open(logfile)) for _ in range(3)]
args = crunchstat_summary.command.ArgumentParser().parse_args(
['--pipeline-instance', self.fake_instance['uuid']])
cmd = crunchstat_summary.command.Command(args)
cmd.run()
- job_report = [
- line for line in open(logfile+'.report').readlines()
- if not line.startswith('#!! ')]
+ with open(logfile+'.report', encoding='utf-8') as f:
+ job_report = [line for line in f if not line.startswith('#!! ')]
expect = (
['### Summary for foo (zzzzz-8i9sb-000000000000000)\n'] +
job_report + ['\n'] +
['### Summary for bar (zzzzz-8i9sb-000000000000001)\n'] +
job_report + ['\n'] +
- ['### Summary for unfinished-job (zzzzz-8i9sb-xxxxxxxxxxxxxxx)\n',
+ ['### Summary for unfinished-job (partial) (zzzzz-8i9sb-xxxxxxxxxxxxxxx)\n',
'(no report generated)\n',
'\n'] +
['### Summary for baz (zzzzz-8i9sb-000000000000002)\n'] +
mock.call('fake-log-pdh-2'),
], any_order=True)
mock_cr().open.assert_called_with('fake-logfile.txt')
+
+
+class SummarizeACRJob(ReportDiff):
+ fake_job = {
+ 'uuid': 'zzzzz-8i9sb-i3e77t9z5y8j9cc',
+ 'owner_uuid': 'zzzzz-tpzed-xurymjxw79nv3jz',
+ 'components': {
+ 'foo': 'zzzzz-8i9sb-000000000000000',
+ 'bar': 'zzzzz-8i9sb-000000000000001',
+ 'unfinished-job': 'zzzzz-8i9sb-xxxxxxxxxxxxxxx',
+ 'baz': 'zzzzz-8i9sb-000000000000002',
+ }
+ }
+ fake_jobs_index = { 'items': [
+ {
+ 'uuid': 'zzzzz-8i9sb-000000000000000',
+ 'log': 'fake-log-pdh-0',
+ 'runtime_constraints': {
+ 'min_ram_mb_per_node': 900,
+ 'min_cores_per_node': 1,
+ },
+ },
+ {
+ 'uuid': 'zzzzz-8i9sb-000000000000001',
+ 'log': 'fake-log-pdh-1',
+ 'runtime_constraints': {
+ 'min_ram_mb_per_node': 900,
+ 'min_cores_per_node': 1,
+ },
+ },
+ {
+ 'uuid': 'zzzzz-8i9sb-xxxxxxxxxxxxxxx',
+ },
+ {
+ 'uuid': 'zzzzz-8i9sb-000000000000002',
+ 'log': 'fake-log-pdh-2',
+ 'runtime_constraints': {
+ 'min_ram_mb_per_node': 900,
+ 'min_cores_per_node': 1,
+ },
+ },
+ ]}
+ @mock.patch('arvados.collection.CollectionReader')
+ @mock.patch('arvados.api')
+ def test_acr_job(self, mock_api, mock_cr):
+ logfile = os.path.join(TESTS_DIR, 'logfile_20151204190335.txt.gz')
+ mock_api().jobs().index().execute.return_value = self.fake_jobs_index
+ mock_api().jobs().get().execute.return_value = self.fake_job
+ mock_cr().__iter__.return_value = ['fake-logfile.txt']
+ mock_cr().open.side_effect = [UTF8Decode(gzip.open(logfile)) for _ in range(3)]
+ args = crunchstat_summary.command.ArgumentParser().parse_args(
+ ['--job', self.fake_job['uuid']])
+ cmd = crunchstat_summary.command.Command(args)
+ cmd.run()
+
+ with open(logfile+'.report', encoding='utf-8') as f:
+ job_report = [line for line in f if not line.startswith('#!! ')]
+ expect = (
+ ['### Summary for zzzzz-8i9sb-i3e77t9z5y8j9cc (partial) (zzzzz-8i9sb-i3e77t9z5y8j9cc)\n',
+ '(no report generated)\n',
+ '\n'] +
+ ['### Summary for bar (zzzzz-8i9sb-000000000000001)\n'] +
+ job_report + ['\n'] +
+ ['### Summary for baz (zzzzz-8i9sb-000000000000002)\n'] +
+ job_report + ['\n'] +
+ ['### Summary for foo (zzzzz-8i9sb-000000000000000)\n'] +
+ job_report + ['\n'] +
+ ['### Summary for unfinished-job (partial) (zzzzz-8i9sb-xxxxxxxxxxxxxxx)\n',
+ '(no report generated)\n']
+ )
+ self.diff_report(cmd, expect)
+ mock_cr.assert_has_calls(
+ [
+ mock.call('fake-log-pdh-0'),
+ mock.call('fake-log-pdh-1'),
+ mock.call('fake-log-pdh-2'),
+ ], any_order=True)
+ mock_cr().open.assert_called_with('fake-logfile.txt')
projects / arvados.git / blobdiff