4 Check lesson files and their contents.
13 from optparse import OptionParser
15 from util import Reporter, read_markdown
19 # Where to look for source Markdown files.
20 SOURCE_DIRS = ['', '_episodes', '_extras']
22 # Required files: each entry is ('path': YAML_required).
23 # FIXME: We do not yet validate whether any files have the required
24 # YAML headers, but should in the future.
25 # The '%' is replaced with the source directory path for checking.
26 # Episodes are handled specially, and extra files in '_extras' are also handled specially.
27 # This list must include all the Markdown files listed in the 'bin/initialize' script.
32 '%/_extras/discuss.md': True,
33 '%/_extras/figures.md': True,
34 '%/_extras/guide.md': True,
36 '%/reference.md': True,
40 # Episode filename pattern.
41 P_EPISODE_FILENAME = re.compile(r'/_episodes/(\d\d)-[-\w]+.md$')
43 # What kinds of blockquotes are allowed?
57 # What kinds of code fragments are allowed?
69 # What fields are required in teaching episode metadata?
70 TEACHING_METADATA_FIELDS = {
79 # What fields are required in break episode metadata?
80 BREAK_METADATA_FIELDS = {
86 # How long are lines allowed to be?
93 args.reporter = Reporter()
94 check_config(args.reporter, args.source_dir)
95 docs = read_all_markdown(args.source_dir, args.parser)
96 check_fileset(args.source_dir, args.reporter, docs.keys())
97 for filename in docs.keys():
98 checker = create_checker(args, filename, docs[filename])
100 args.reporter.report()
104 """Parse command-line arguments."""
106 parser = OptionParser()
107 parser.add_option('-l', '--linelen',
110 help='Check line lengths')
111 parser.add_option('-p', '--parser',
114 help='path to Markdown parser')
115 parser.add_option('-s', '--source',
118 help='source directory')
120 args, extras = parser.parse_args()
121 require(args.parser is not None,
122 'Path to Markdown parser not provided')
124 'Unexpected trailing command-line arguments "{0}"'.format(extras))
129 def check_config(reporter, source_dir):
130 """Check configuration file."""
132 config_file = os.path.join(source_dir, '_config.yml')
133 with open(config_file, 'r') as reader:
134 config = yaml.load(reader)
135 reporter.check_field(config_file, 'configuration', config, 'kind', 'lesson')
138 def read_all_markdown(source_dir, parser):
139 """Read source files, returning
140 {path : {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}}
143 all_dirs = [os.path.join(source_dir, d) for d in SOURCE_DIRS]
144 all_patterns = [os.path.join(d, '*.md') for d in all_dirs]
146 for pat in all_patterns:
147 for filename in glob.glob(pat):
148 data = read_markdown(parser, filename)
150 result[filename] = data
154 def check_fileset(source_dir, reporter, filenames_present):
155 """Are all required files present? Are extraneous files present?"""
157 # Check files with predictable names.
158 required = [p.replace('%', source_dir) for p in REQUIRED_FILES]
159 missing = set(required) - set(filenames_present)
161 reporter.add(None, 'Missing required file {0}', m)
163 # Check episode files' names.
165 for filename in filenames_present:
166 if '_episodes' not in filename:
168 m = P_EPISODE_FILENAME.search(filename)
170 seen.append(m.group(1))
172 reporter.add(None, 'Episode {0} has badly-formatted filename', filename)
174 # Check for duplicate episode numbers.
175 reporter.check(len(seen) == len(set(seen)),
177 'Duplicate episode numbers {0} vs {1}',
178 sorted(seen), sorted(set(seen)))
180 # Check that numbers are consecutive.
181 seen = [int(s) for s in seen]
184 for i in range(len(seen) - 1):
185 clean = clean and ((seen[i+1] - seen[i]) == 1)
186 reporter.check(clean,
188 'Missing or non-consecutive episode numbers {0}',
192 def create_checker(args, filename, info):
193 """Create appropriate checker for file."""
195 for (pat, cls) in CHECKERS:
196 if pat.search(filename):
197 return cls(args, filename, **info)
200 def require(condition, message):
201 """Fail if condition not met."""
204 print(message, file=sys.stderr)
208 class CheckBase(object):
209 """Base class for checking Markdown files."""
211 def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
212 """Cache arguments for checking."""
214 super(CheckBase, self).__init__()
216 self.reporter = self.args.reporter # for convenience
217 self.filename = filename
218 self.metadata = metadata
219 self.metadata_len = metadata_len
228 """Run tests on metadata."""
230 self.check_metadata()
232 self.check_blockquote_classes()
233 self.check_codeblock_classes()
236 def check_metadata(self):
237 """Check the YAML metadata."""
239 self.reporter.check(self.metadata is not None,
241 'Missing metadata entirely')
243 if self.metadata and (self.layout is not None):
244 self.reporter.check_field(self.filename, 'metadata', self.metadata, 'layout', self.layout)
247 def check_text(self):
248 """Check the raw text of the lesson body."""
250 if self.args.line_len:
251 over = [i for (i, l, n) in self.lines if (n > MAX_LINE_LEN) and (not l.startswith('!'))]
252 self.reporter.check(not over,
254 'Line(s) are too long: {0}',
255 ', '.join([str(i) for i in over]))
258 def check_blockquote_classes(self):
259 """Check that all blockquotes have known classes."""
261 for node in self.find_all(self.doc, {'type' : 'blockquote'}):
262 cls = self.get_val(node, 'attr', 'class')
263 self.reporter.check(cls in KNOWN_BLOCKQUOTES,
264 (self.filename, self.get_loc(node)),
265 'Unknown or missing blockquote type {0}',
269 def check_codeblock_classes(self):
270 """Check that all code blocks have known classes."""
272 for node in self.find_all(self.doc, {'type' : 'codeblock'}):
273 cls = self.get_val(node, 'attr', 'class')
274 self.reporter.check(cls in KNOWN_CODEBLOCKS,
275 (self.filename, self.get_loc(node)),
276 'Unknown or missing code block type {0}',
280 def find_all(self, node, pattern, accum=None):
281 """Find all matches for a pattern."""
283 assert type(pattern) == dict, 'Patterns must be dictionaries'
286 if self.match(node, pattern):
288 for child in node.get('children', []):
289 self.find_all(child, pattern, accum)
293 def match(self, node, pattern):
294 """Does this node match the given pattern?"""
303 elif type(val) == dict:
304 if not self.match(node[key], val):
309 def get_val(self, node, *chain):
310 """Get value one or more levels down."""
313 for selector in chain:
314 curr = curr.get(selector, None)
320 def get_loc(self, node):
321 """Convenience method to get node's line number."""
323 result = self.get_val(node, 'options', 'location')
324 if self.metadata_len is not None:
325 result += self.metadata_len
329 class CheckNonJekyll(CheckBase):
330 """Check a file that isn't translated by Jekyll."""
332 def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
333 super(CheckNonJekyll, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)
336 def check_metadata(self):
337 self.reporter.check(self.metadata is None,
339 'Unexpected metadata')
342 class CheckIndex(CheckBase):
343 """Check the main index page."""
345 def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
346 super(CheckIndex, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)
347 self.layout = 'lesson'
350 class CheckEpisode(CheckBase):
351 """Check an episode page."""
353 def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
354 super(CheckEpisode, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)
356 def check_metadata(self):
357 super(CheckEpisode, self).check_metadata()
359 if 'layout' in self.metadata:
360 if self.metadata['layout'] == 'break':
361 self.check_metadata_fields(BREAK_METADATA_FIELDS)
363 self.reporter.add(self.filename,
364 'Unknown episode layout "{0}"',
365 self.metadata['layout'])
367 self.check_metadata_fields(TEACHING_METADATA_FIELDS)
370 def check_metadata_fields(self, expected):
371 for (name, type_) in expected:
372 if name not in self.metadata:
373 self.reporter.add(self.filename,
374 'Missing metadata field {0}',
376 elif type(self.metadata[name]) != type_:
377 self.reporter.add(self.filename,
378 '"{0}" has wrong type in metadata ({1} instead of {2})',
379 name, type(self.metadata[name]), type_)
382 class CheckReference(CheckBase):
383 """Check the reference page."""
385 def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
386 super(CheckReference, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)
387 self.layout = 'reference'
390 class CheckGeneric(CheckBase):
391 """Check a generic page."""
393 def __init__(self, args, filename, metadata, metadata_len, text, lines, doc):
394 super(CheckGeneric, self).__init__(args, filename, metadata, metadata_len, text, lines, doc)
399 (re.compile(r'CONTRIBUTING\.md'), CheckNonJekyll),
400 (re.compile(r'README\.md'), CheckNonJekyll),
401 (re.compile(r'index\.md'), CheckIndex),
402 (re.compile(r'reference\.md'), CheckReference),
403 (re.compile(r'_episodes/.*\.md'), CheckEpisode),
404 (re.compile(r'.*\.md'), CheckGeneric)
408 if __name__ == '__main__':