Fix raw html reference issue (Python-Markdown#585)

Preserve the line which a reference was on to prevent raw HTML indexing issue. Fixes Python-Markdown#584. Prevent raw HTML parsing issue in abbr and footnotes Peserve abbreviation line when stripping and preserve a line for each footnote block. Footnotes should also accumulate the extraneous padding. Test extra lines at the end of references Strip the gathered extraneous whitespace When processing footnotes, we don't actually care to process the extra whitespace at the end of a footnote, but we want it to calculate lines to preserve.
atalw · Jan 4, 2018 · 1de595a · 1de595a
1 parent bbada79
commit 1de595a
Show file tree

Hide file tree

Showing 5 changed files with 203 additions and 1 deletion.
diff --git a/markdown/extensions/abbr.py b/markdown/extensions/abbr.py
@@ -53,6 +53,9 @@ def run(self, lines):
                 title = m.group('title').strip()
                 self.markdown.inlinePatterns['abbr-%s' % abbr] = \
                     AbbrPattern(self._generate_pattern(abbr), title)
+                # Preserve the line to prevent raw HTML indexing issue.
+                # https://github.com/Python-Markdown/markdown/issues/584
+                new_text.append('')
             else:
                 new_text.append(line)
         return new_text

diff --git a/markdown/extensions/footnotes.py b/markdown/extensions/footnotes.py
@@ -238,7 +238,12 @@ def run(self, lines):
                 fn, _i = self.detectTabbed(lines[i+1:])
                 fn.insert(0, m.group(2))
                 i += _i-1  # skip past footnote
-                self.footnotes.setFootnote(m.group(1), "\n".join(fn))
+                footnote = "\n".join(fn)
+                self.footnotes.setFootnote(m.group(1), footnote.rstrip())
+                # Preserve a line for each block to prevent raw HTML indexing issue.
+                # https://github.com/Python-Markdown/markdown/issues/584
+                num_blocks = (len(footnote.split('\n\n')) * 2)
+                newlines.extend([''] * (num_blocks))
             else:
                 newlines.append(lines[i])
             if len(lines) > i+1:
@@ -290,6 +295,11 @@ def detab(line):
                     if lines[j].strip():
                         next_line = lines[j]
                         break
+                    else:
+                        # Include extreaneous padding to prevent raw HTML
+                        # parsing issue: https://github.com/Python-Markdown/markdown/issues/584
+                        items.append("")
+                        i += 1
                 else:
                     break  # There is no more text; we are done.
 

diff --git a/markdown/preprocessors.py b/markdown/preprocessors.py
@@ -346,6 +346,9 @@ def run(self, lines):
                         lines.pop(0)
                         t = tm.group(2) or tm.group(3) or tm.group(4)
                 self.markdown.references[id] = (link, t)
+                # Preserve the line to prevent raw HTML indexing issue.
+                # https://github.com/Python-Markdown/markdown/issues/584
+                new_text.append('')
             else:
                 new_text.append(line)
 

diff --git a/tests/extensions/extra/raw-html.html b/tests/extensions/extra/raw-html.html
@@ -46,4 +46,79 @@
 <p>foo bar</p>
 <p><em>bar</em>
 </p>
+</div>
+<div name="issue584">
+<div>
+<p><a href="http://example.com">link</a></p>
+</div>
+</div>
+<div name="issue584">
+<div>
+<p><abbr title="Abbreviation">abbr</abbr></p>
+</div>
+</div>
+<div name="issue584">
+<div>
+<p>footnote<sup id="fnref:1"><a class="footnote-ref" href="#fn:1" rel="footnote">1</a></sup></p>
+</div>
+</div>
+<div name="issue584">
+<div>
+<p><a href="http://example.com">link</a></p>
+</div>
+</div>
+<div name="issue584">
+<div>
+<p><abbr title="Abbreviation">abbr</abbr></p>
+</div>
+</div>
+<div name="issue584">
+<div>
+<p>footnote<sup id="fnref:2"><a class="footnote-ref" href="#fn:2" rel="footnote">2</a></sup></p>
+</div>
+</div>
+<div class="footnote">
+<hr />
+<ol>
+<li id="fn:1">
+<ol>
+<li>
+<p>The top couple half figure, contrary sides and hands across with bottom couple,</p>
+<p>Half figure back on your own sides, and turn partner to places,</p>
+<p>Swing partners with right hands into straight line long-ways, as in a reel, and</p>
+<p>Set,</p>
+<p>Hey and return to places,</p>
+<p>The other three couples do the same.</p>
+</li>
+<li>
+<p>Top and bottom couples meet and set,</p>
+<p>Then each gentleman leas the opposite lady to the couple on his left, and set,</p>
+<p>Aach four right and left,</p>
+<p>Swing side couples to places, and turn partners all eight,</p>
+<p>The other two couple o the same.</p>
+</li>
+</ol>
+<p><a class="footnote-backref" href="#fnref:1" rev="footnote" title="Jump back to footnote 1 in the text">&#8617;</a></p>
+</li>
+<li id="fn:2">
+<ol>
+<li>
+<p>The top couple half figure, contrary sides and hands across with bottom couple,</p>
+<p>Half figure back on your own sides, and turn partner to places,</p>
+<p>Swing partners with right hands into straight line long-ways, as in a reel, and</p>
+<p>Set,</p>
+<p>Hey and return to places,</p>
+<p>The other three couples do the same.</p>
+</li>
+<li>
+<p>Top and bottom couples meet and set,</p>
+<p>Then each gentleman leas the opposite lady to the couple on his left, and set,</p>
+<p>Aach four right and left,</p>
+<p>Swing side couples to places, and turn partners all eight,</p>
+<p>The other two couple o the same.</p>
+</li>
+</ol>
+<p><a class="footnote-backref" href="#fnref:2" rev="footnote" title="Jump back to footnote 2 in the text">&#8617;</a></p>
+</li>
+</ol>
 </div>
diff --git a/tests/extensions/extra/raw-html.txt b/tests/extensions/extra/raw-html.txt
@@ -71,3 +71,114 @@ foo bar
 
 <em>bar</em>
 </div>
+
+<div markdown="1" name="issue584">
+
+[link]: http://example.com
+
+<div markdown="1">
+[link][link]
+</div>
+
+</div>
+
+<div markdown="1" name="issue584">
+
+*[abbr]: Abbreviation
+
+<div markdown="1">
+abbr
+</div>
+
+</div>
+
+<div markdown="1" name="issue584">
+
+[^1]: 
+    1.  The top couple half figure, contrary sides and hands across with bottom couple,
+
+        Half figure back on your own sides, and turn partner to places,
+
+        Swing partners with right hands into straight line long-ways, as in a reel, and
+
+        Set,
+
+        Hey and return to places,
+
+        The other three couples do the same.
+
+    2.  Top and bottom couples meet and set,
+
+        Then each gentleman leas the opposite lady to the couple on his left, and set,
+
+        Aach four right and left,
+
+        Swing side couples to places, and turn partners all eight,
+
+        The other two couple o the same.
+
+<div markdown="1">
+footnote[^1]
+</div>
+
+</div>
+
+<div markdown="1" name="issue584">
+
+[link]: http://example.com
+
+
+
+
+<div markdown="1">
+[link][link]
+</div>
+
+</div>
+
+<div markdown="1" name="issue584">
+
+*[abbr]: Abbreviation
+
+
+
+
+<div markdown="1">
+abbr
+</div>
+
+</div>
+
+<div markdown="1" name="issue584">
+
+[^2]: 
+    1.  The top couple half figure, contrary sides and hands across with bottom couple,
+
+        Half figure back on your own sides, and turn partner to places,
+
+        Swing partners with right hands into straight line long-ways, as in a reel, and
+
+        Set,
+
+        Hey and return to places,
+
+        The other three couples do the same.
+
+    2.  Top and bottom couples meet and set,
+
+        Then each gentleman leas the opposite lady to the couple on his left, and set,
+
+        Aach four right and left,
+
+        Swing side couples to places, and turn partners all eight,
+
+        The other two couple o the same.
+
+
+
+
+<div markdown="1">
+footnote[^2]
+</div>
+
+</div>