Changeset 42 for bitstructures

Timestamp:

01/06/08 14:41:54 (12 months ago)

Author:

simon

Message:

implemented syntax highlighting for codeblocks

Location:

bitstructures/trunk

Files:

• README (modified) (1 diff)
• css/bitstructures.css (modified) (1 diff)
• substructure/codeblocks.py (modified) (4 diffs)
• substructure/templatetags/substructure_tags.py (modified) (2 diffs)
• substructure/tests.py (modified) (4 diffs)
• substructure/views.py (modified) (1 diff)
• templates/substructure/entry.html (modified) (2 diffs)

bitstructures/trunk/README

@@ -13 +13 @@
 Django 0.96
 Python Markdown 1.6a http://www.freewisdom.org/projects/python-markdown/
+Pygments 0.9 http://pygments.org/

bitstructures/trunk/css/bitstructures.css

@@ -120 +120 @@
         text-decoration: none;
 }
+
+pre {
+        background-color: #def;
+}
+
+/* rules for Pygments */
+
+.highlight .k, .highlight .nb, .highlight .nt {
+        color: #555;
+        font-weight: bold;
+}

bitstructures/trunk/substructure/codeblocks.py

@@ -18 +18 @@
             return self.text
 
+def is_codeblock_header_field(line):
+    if line.is_codeblock_line() and get_field_name(line):
+        return True
+    else:
+        return False
+
+def get_field_name(header_line):
+    name_value = header_line.get_text_without_indentation()
+    index = name_value.find(':')
+    if index != -1:
+        return name_value[0:index].strip()
+    else:
+        return None
+
+def get_field_value(header_line):
+    name_value = header_line.get_text_without_indentation()
+    index = name_value.find(':')
+    if index != -1:
+        return name_value[index+1:].strip()
+    else:
+        return None
+
 class MarkdownLineTokeniser:
     def __init__(self, text):
         self._lines = iter(text.splitlines(True))
-        self._current_line = None
+        self.current_line = None
         self._at_end = False
-
-    def get_current_line(self):
-        return self._current_line
 
     def is_at_end(self):
@@ -32 +51 @@
     def read_next(self):
         try:
-            self._current_line = MarkdownLine(self._lines.next())
+            self.current_line = MarkdownLine(self._lines.next())
         except StopIteration:
             self._at_end = True
-            self._current_line = None
+            self.current_line = None
+
+class Section:
+    def __init__(self):
+        self.lines = list()
+
+    def get_text(self):
+        return ''.join(line.text for line in self.lines)
+
+    def append(self, line):
+        self.lines.append(line)
+
+class Codeblock(Section):
+    def __init__(self):
+        Section.__init__(self)
+        self.content_type = None
+
+    def is_codeblock(self):
+        return True
+
+    def get_code(self):
+        return ''.join(line.get_text_without_indentation() for line in self.lines)
+
+class NonCodeblock(Section):
+    def is_codeblock(self):
+        return False
 
 class MarkdownCodeblocksParser:
@@ -45 +89 @@
         lines.read_next()
         while not lines.is_at_end():
-            if lines.get_current_line().is_indented():
-                codeblock = self._get_codeblock_text(lines)
+            if lines.current_line.is_indented():
+                codeblock = self._get_codeblock(lines)
                 count += 1
                 if count == n:
@@ -53 +97 @@
         return None
 
-    def _get_codeblock_text(self, lines):
-        text = str()
-        while (not lines.is_at_end()) and lines.get_current_line().is_codeblock_line():
-            text = text + lines.get_current_line().get_text_without_indentation()
+    def parse(self, text):
+        sections = list()
+        lines = MarkdownLineTokeniser(text)
+        lines.read_next()
+        while not lines.is_at_end():
+            if lines.current_line.is_indented():
+                sections.append(self._get_codeblock(lines))
+            else:
+                sections.append(self._get_non_codeblock(lines))
+        return sections
+
+    def _get_codeblock(self, lines):
+        codeblock = Codeblock()
+        if is_codeblock_header_field(lines.current_line):
+            if get_field_name(lines.current_line) == 'Content-Type':
+                codeblock.content_type = get_field_value(lines.current_line)
             lines.read_next()
-        return text
+        while (not lines.is_at_end()) and lines.current_line.is_codeblock_line():
+            codeblock.append(lines.current_line)
+            lines.read_next()
+        return codeblock
+
+    def _get_non_codeblock(self, lines):
+        section = NonCodeblock()
+        while (not lines.is_at_end()) and (not lines.current_line.is_indented()):
+            section.append(lines.current_line)
+            lines.read_next()
+        return section

bitstructures/trunk/substructure/templatetags/substructure_tags.py

@@ -1 +1 @@
 from django import template
 from django.utils.tzinfo import LocalTimezone
+from markdown import markdown
+from pygments import highlight
+from pygments.lexers import get_lexer_for_mimetype
+from pygments.formatters import HtmlFormatter
+from bitstructures.substructure.codeblocks import MarkdownCodeblocksParser
 
 def rfc3339(dt):
@@ -9 +14 @@
     return dt_out.isoformat()
 
+def syntax_highlighted_markdown(markdown_text):
+    parser = MarkdownCodeblocksParser()
+    formatter = HtmlFormatter()
+    sections = parser.parse(markdown_text)
+    for_markdown = list()
+    html = list()
+    for section in sections:
+        if section.is_codeblock() and section.content_type:
+            # run Markdown on any text that isn't to be syntax highlighted
+            if for_markdown:
+                html.append(markdown(''.join(for_markdown)))
+                for_markdown = list()
+            # run Pygments on code to be syntax highlighted
+            html.append(highlight(section.get_code(),
+                get_lexer_for_mimetype(section.content_type), formatter))
+        else:
+            for_markdown.append(section.get_text())
+    # run Markdown on any remaining text that isn't to be syntax highlighted
+    if for_markdown:
+        html.append(markdown(''.join(for_markdown)))
+    return ''.join(html)
+
 register = template.Library()
-register.filter("rfc3339", rfc3339)
+register.filter('rfc3339', rfc3339)
+register.filter('syntax_highlighted_markdown', syntax_highlighted_markdown)

bitstructures/trunk/substructure/tests.py

@@ -1 @@
-from bitstructures.substructure.codeblocks import MarkdownLine, MarkdownCodeblocksParser
+from bitstructures.substructure.codeblocks import MarkdownLine, MarkdownCodeblocksParser, is_codeblock_header_field, get_field_name, get_field_value
 import unittest
 
 class MarkdownLineTest(unittest.TestCase):
+    def setUp(self):
+        self.empty = MarkdownLine('')
+        self.foo_bar = MarkdownLine('    foo:bar')
+        self.foo_bar_spaces = MarkdownLine('    foo : bar')
+        self.foo_bar_newline = MarkdownLine('    foo:bar\n')
+
     def test_is_blank(self):
-        self.assert_(MarkdownLine('').is_blank())
+        self.assert_(self.empty.is_blank())
         self.assert_(MarkdownLine(' ').is_blank())
         self.assert_(MarkdownLine('\t').is_blank())
@@ -14 +20 @@
         self.assert_(MarkdownLine('    code').is_indented())
         self.assert_(MarkdownLine('     code').is_indented())
+
+    def test_codeblock_header_field(self):
+        self.assert_(not is_codeblock_header_field(self.empty))
+        self.assert_(not is_codeblock_header_field(MarkdownLine('foo')))
+        self.assert_(not is_codeblock_header_field(MarkdownLine('foo:bar')))
+        self.assert_(not is_codeblock_header_field(MarkdownLine('    foo')))
+        self.assert_(not is_codeblock_header_field(MarkdownLine('    :')))
+        self.assert_(not is_codeblock_header_field(MarkdownLine('    :foo')))
+        self.assert_(is_codeblock_header_field(self.foo_bar))
+        self.assertEqual('foo', get_field_name(self.foo_bar))
+        self.assertEqual('bar', get_field_value(self.foo_bar))
+        self.assert_(is_codeblock_header_field(self.foo_bar_spaces))
+        self.assertEqual('foo', get_field_name(self.foo_bar_spaces))
+        self.assertEqual('bar', get_field_value(self.foo_bar_spaces))
+        self.assert_(is_codeblock_header_field(self.foo_bar_newline))
+        self.assertEqual('foo', get_field_name(self.foo_bar_newline))
+        self.assertEqual('bar', get_field_value(self.foo_bar_newline))
 
 class MarkdownCodeblocksParserTest(unittest.TestCase):
@@ -33 +56 @@
 text
     code2'''
+        self.with_content_type = '''    Content-Type: text/x-python
+    line2'''
 
     def test_get_codeblock_on_empty(self):
@@ -52 +77 @@
 
     def test_get_codeblock(self):
-        self.assertEqual('foo', self.parser.get_codeblock('    foo', 1))
+        self.assertEqual('    foo', self.parser.get_codeblock('    foo', 1).get_text())
+        self.assertEqual('foo', self.parser.get_codeblock('    foo', 1).get_code())
+        self.assertEqual(self.two_line,
+            self.parser.get_codeblock(self.two_line, 1).get_text())
         expect_twoline = '''line1
 line2'''
         self.assertEqual(expect_twoline,
-                         self.parser.get_codeblock(self.two_line, 1))
+            self.parser.get_codeblock(self.two_line, 1).get_code())
+        self.assertEqual('    code\n\n',
+            self.parser.get_codeblock(self.text_code_text, 1).get_text())
         self.assertEqual('code\n\n',
-                         self.parser.get_codeblock(self.text_code_text, 1))
+            self.parser.get_codeblock(self.text_code_text, 1).get_code())
+        self.assertEqual('    code1\n\n',
+            self.parser.get_codeblock(self.code_text_code, 1).get_text())
         self.assertEqual('code1\n\n',
-                         self.parser.get_codeblock(self.code_text_code, 1))
+            self.parser.get_codeblock(self.code_text_code, 1).get_code())
+        self.assertEqual('    code2\n',
+            self.parser.get_codeblock(self.code_text_code, 2).get_text())
         self.assertEqual('code2\n',
-                         self.parser.get_codeblock(self.code_text_code, 2))
+            self.parser.get_codeblock(self.code_text_code, 2).get_code())
+        self.assertEqual('    code1\n',
+            self.parser.get_codeblock(self.code_text_code_no_blank_lines, 1).get_text())
         self.assertEqual('code1\n',
-                         self.parser.get_codeblock(self.code_text_code_no_blank_lines, 1))
+            self.parser.get_codeblock(self.code_text_code_no_blank_lines, 1).get_code())
+        self.assertEqual('    code2',
+            self.parser.get_codeblock(self.code_text_code_no_blank_lines, 2).get_text())
         self.assertEqual('code2',
-                         self.parser.get_codeblock(self.code_text_code_no_blank_lines, 2))
+            self.parser.get_codeblock(self.code_text_code_no_blank_lines, 2).get_code())
+
+    def test_get_codeblock_with_content_type(self):
+        self.assertEqual('text/x-python',
+            self.parser.get_codeblock(self.with_content_type, 1).content_type)
+        self.assertEqual('    line2',
+            self.parser.get_codeblock(self.with_content_type, 1).get_text())
+        self.assertEqual('line2',
+            self.parser.get_codeblock(self.with_content_type, 1).get_code())
+
+    def test_parse_text_code_text(self):
+        sections = self.parser.parse(self.text_code_text)
+        self.assertEqual(3, len(sections))
+        self.assert_(not sections[0].is_codeblock())
+        self.assertEqual('text\n\n', sections[0].get_text())
+        self.assert_(sections[1].is_codeblock())
+        self.assertEqual('    code\n\n', sections[1].get_text())
+        self.assertEqual('code\n\n', sections[1].get_code())
+        self.assert_(not sections[2].is_codeblock())
+        self.assertEqual('text', sections[2].get_text())
+
+    def test_parse_code_text_code(self):
+        sections = self.parser.parse(self.code_text_code)
+        self.assertEqual(3, len(sections))
+        self.assert_(sections[0].is_codeblock())
+        self.assertEqual('    code1\n\n', sections[0].get_text())
+        self.assertEqual('code1\n\n', sections[0].get_code())
+        self.assert_(not sections[1].is_codeblock())
+        self.assertEqual('text\n\n', sections[1].get_text())
+        self.assert_(sections[2].is_codeblock())
+        self.assertEqual('    code2\n', sections[2].get_text())
+        self.assertEqual('code2\n', sections[2].get_code())

bitstructures/trunk/substructure/views.py

@@ -53 +53 @@
     if codeblock:
         if filename.lower().endswith('html'):
-            return HttpResponse(codeblock, mimetype='text/html')
+            return HttpResponse(codeblock.get_code(), mimetype='text/html')
         else:
-            return HttpResponse(codeblock, mimetype='text/plain')
+            return HttpResponse(codeblock.get_code(), mimetype='text/plain')
     else:
         raise Http404

bitstructures/trunk/templates/substructure/entry.html

@@ -1 @@
-{% load django.contrib.markup %}
+{% load substructure_tags %}
 
 <div class="entry">
@@ -13 +13 @@
 {% endif %}
 
-{{ entry.text|markdown }}
+{{ entry.text|syntax_highlighted_markdown }}
 
 </div>