bugdown: Trigger test failure for invalid Markdown include statements.

This commit adds a custom Markdown include extension which is
identical to the original except when a macro file can't
be found, it raises a custom JsonableError exception, which
we can catch and then trigger an appropriate test failure.

Fixes: zulip#10947

templates/zerver/tests/markdown/test_custom_include_extension.md

@@ -0,0 +1 @@
+{!nonexistent-macro.md!}

templates/zerver/tests/markdown/test_custom_include_extension_empty.md

@@ -0,0 +1 @@
+{!empty.md!}

zerver/lib/bugdown/include.py

@@ -0,0 +1,67 @@
+from __future__ import print_function
+import re
+import os
+from typing import Any, Dict, Optional, List
+
+import markdown
+from markdown_include.include import MarkdownInclude, IncludePreprocessor
+
+from zerver.lib.exceptions import InvalidMarkdownIncludeStatement
+
+INC_SYNTAX = re.compile(r'\{!\s*(.+?)\s*!\}')
+
+
+class MarkdownIncludeCustom(MarkdownInclude):
+    def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
+        md.preprocessors.add(
+            'include_wrapper',
+            IncludeCustomPreprocessor(md, self.getConfigs()),
+            '_begin'
+        )
+
+class IncludeCustomPreprocessor(IncludePreprocessor):
+    """
+    This is a custom implementation of the markdown_include
+    extension that checks for include statements and if the included
+    macro file does not exist or can't be opened, raises a custom
+    JsonableError exception. The rest of the functionality is identical
+    to the original markdown_include extension.
+    """
+    def run(self, lines: List[str]) -> List[str]:
+        done = False
+        while not done:
+            for line in lines:
+                loc = lines.index(line)
+                m = INC_SYNTAX.search(line)
+
+                if m:
+                    filename = m.group(1)
+                    filename = os.path.expanduser(filename)
+                    if not os.path.isabs(filename):
+                        filename = os.path.normpath(
+                            os.path.join(self.base_path, filename)
+                        )
+                    try:
+                        with open(filename, 'r', encoding=self.encoding) as r:
+                            text = r.readlines()
+                    except Exception as e:
+                        print('Warning: could not find file {}. Error: {}'.format(filename, e))
+                        lines[loc] = INC_SYNTAX.sub('', line)
+                        raise InvalidMarkdownIncludeStatement(m.group(0).strip())
+
+                    line_split = INC_SYNTAX.split(line)
+                    if len(text) == 0:
+                        text.append('')
+                    for i in range(len(text)):
+                        text[i] = text[i].rstrip('\r\n')
+                    text[0] = line_split[0] + text[0]
+                    text[-1] = text[-1] + line_split[2]
+                    lines = lines[:loc] + text + lines[loc+1:]
+                    break
+            else:
+                done = True
+
+        return lines
+
+def makeExtension(*args: Any, **kwargs: str) -> MarkdownIncludeCustom:
+    return MarkdownIncludeCustom(kwargs)

zerver/lib/exceptions.py

@@ -41,6 +41,7 @@ class ErrorCode(AbstractEnum):
     CSRF_FAILED = ()
     INVITATION_FAILED = ()
     INVALID_ZULIP_SERVER = ()
+    INVALID_MARKDOWN_INCLUDE_STATEMENT = ()
     REQUEST_CONFUSING_VAR = ()
 
 class JsonableError(Exception):
@@ -152,6 +153,17 @@ def __init__(self, is_last_admin: bool) -> None:
     def msg_format() -> str:
         return _("Cannot deactivate the only {entity}.")
 
+class InvalidMarkdownIncludeStatement(JsonableError):
+    code = ErrorCode.INVALID_MARKDOWN_INCLUDE_STATEMENT
+    data_fields = ['include_statement']
+
+    def __init__(self, include_statement: str) -> None:
+        self.include_statement = include_statement
+
+    @staticmethod
+    def msg_format() -> str:
+        return _("Invalid markdown include statement: {include_statement}")
+
 class RateLimited(PermissionDenied):
     def __init__(self, msg: str="") -> None:
         super().__init__(msg)

zerver/templatetags/app_filters.py

@@ -7,7 +7,6 @@
 import markdown.extensions.codehilite
 import markdown.extensions.extra
 import markdown.extensions.toc
-import markdown_include.include
 from django.conf import settings
 from django.template import Library, engines, loader
 from django.utils.safestring import mark_safe
@@ -21,6 +20,7 @@
 import zerver.lib.bugdown.help_settings_links
 import zerver.lib.bugdown.help_relative_links
 import zerver.lib.bugdown.help_emoticon_translations_table
+import zerver.lib.bugdown.include
 from zerver.context_processors import zulip_default_context
 from zerver.lib.cache import ignore_unhashable_lru_cache
 
@@ -115,7 +115,7 @@ def render_markdown_path(markdown_file_path: str,
             zerver.lib.bugdown.help_emoticon_translations_table.makeExtension(),
         ]
     if md_macro_extension is None:
-        md_macro_extension = markdown_include.include.makeExtension(
+        md_macro_extension = zerver.lib.bugdown.include.makeExtension(
             base_path='templates/zerver/help/include/')
 
     if any(doc in markdown_file_path for doc in docs_without_macros):

zerver/tests/test_templates.py

@@ -11,6 +11,7 @@
 from django.template.loader import get_template
 from django.test.client import RequestFactory
 
+from zerver.lib.exceptions import InvalidMarkdownIncludeStatement
 from zerver.lib.test_helpers import get_all_templates
 from zerver.lib.test_classes import (
     ZulipTestCase,
@@ -312,6 +313,25 @@ def test_markdown_nested_code_blocks(self) -> None:
                     'non-indentedcodeblockwithmultiplelines</pre></div>footer')
         self.assertEqual(content_sans_whitespace, expected)
 
+    def test_custom_markdown_include_extension(self) -> None:
+        template = get_template("tests/test_markdown.html")
+        context = {
+            'markdown_test_file': "zerver/tests/markdown/test_custom_include_extension.md"
+        }
+
+        with self.assertRaisesRegex(InvalidMarkdownIncludeStatement, "Invalid markdown include statement"):
+            template.render(context)
+
+    def test_custom_markdown_include_extension_empty_macro(self) -> None:
+        template = get_template("tests/test_markdown.html")
+        context = {
+            'markdown_test_file': "zerver/tests/markdown/test_custom_include_extension_empty.md"
+        }
+        content = template.render(context)
+        content_sans_whitespace = content.replace(" ", "").replace('\n', '')
+        expected = 'headerfooter'
+        self.assertEqual(content_sans_whitespace, expected)
+
     def test_custom_tos_template(self) -> None:
         response = self.client_get("/terms/")