👌 IMPROVE: Store rawtext in AST nodes (for gettext) (#301)

In this commit we ensure that the raw text is propagated
from the Markdown tokens to the Sphinx AST.
In particular, this is used by the `gettext` builder,
to generate translation POT templates.

Author: jpmckinney

myst_parser/docutils_renderer.py

@@ -325,7 +325,7 @@ def renderInlineAsText(self, tokens: List[Token]) -> str:
     # ### render methods for commonmark tokens
 
     def render_paragraph_open(self, token: NestedTokens):
-        para = nodes.paragraph("")
+        para = nodes.paragraph(token.children[0].content if token.children else "")
         self.add_line_and_source_path(para, token)
         with self.current_node_context(para, append=True):
             self.render_children(token)
@@ -449,7 +449,7 @@ def render_heading_open(self, token: NestedTokens):
             if self.is_section_level(level, self.current_node):
                 self.current_node = cast(nodes.Element, self.current_node.parent)
 
-        title_node = nodes.title()
+        title_node = nodes.title(token.children[0].content if token.children else "")
         self.add_line_and_source_path(title_node, token)
 
         new_section = nodes.section()
@@ -719,7 +719,9 @@ def render_table_row(self, token: Token):
         with self.current_node_context(row, append=True):
             for child in token.children or []:
                 entry = nodes.entry()
-                para = nodes.paragraph("")
+                para = nodes.paragraph(
+                    child.children[0].content if child.children else ""
+                )
                 style = child.attrGet("style")  # i.e. the alignment when using e.g. :--
                 if style:
                     entry["classes"].append(style)
@@ -865,7 +867,9 @@ def render_dl_open(self, token: NestedTokens):
                     item = nodes.definition_list_item()
                     self.add_line_and_source_path(item, child)
                     with self.current_node_context(item, append=True):
-                        term = nodes.term()
+                        term = nodes.term(
+                            child.children[0].content if child.children else ""
+                        )
                         self.add_line_and_source_path(term, child)
                         with self.current_node_context(term, append=True):
                             self.render_children(child)

tests/test_sphinx/sourcedirs/gettext/conf.py

@@ -0,0 +1,4 @@
+extensions = ["myst_parser"]
+exclude_patterns = ["_build"]
+copyright = "2020, Executable Book Project"
+myst_enable_extensions = ["deflist"]

tests/test_sphinx/sourcedirs/gettext/index.md

@@ -0,0 +1,42 @@
+# **bold** text 1
+
+**bold** text 2
+
+> **bold** text 3
+
+```{eval-rst}
+.. note::
+
+   **bold** text 4
+```
+
+* **bold** text 5
+
+1. **bold** text 6
+
+**bold** text 7
+: **bold** text 8
+
+| **bold** text 9  |
+| ---------------- |
+| **bold** text 10 |
+
+<div markdown=1>
+
+**bold** text 11
+
+</div>
+
+    **skip** text
+
+```
+**skip** text
+```
+
+```json
+{
+    "skip": "text"
+}
+```
+
+<h3>**skip** text</h3>

tests/test_sphinx/test_sphinx_builds.py

@@ -3,6 +3,7 @@
 see conftest.py for fixture usage
 """
 import os
+import re
 
 import pytest
 
@@ -272,3 +273,26 @@ def test_substitutions(
         )
     finally:
         get_sphinx_app_output(app, filename="index.html", regress_html=True)
+
+
+@pytest.mark.sphinx(
+    buildername="gettext", srcdir=os.path.join(SOURCE_DIR, "gettext"), freshenv=True
+)
+def test_gettext(
+    app,
+    status,
+    warning,
+    get_sphinx_app_output,
+    remove_sphinx_builds,
+    file_regression,
+):
+    """Test gettext message extraction."""
+    app.build()
+    assert "build succeeded" in status.getvalue()  # Build succeeded
+    warnings = warning.getvalue().strip()
+    assert warnings == ""
+
+    output = get_sphinx_app_output(app, filename="index.pot", buildername="gettext")
+    output = re.sub(r"POT-Creation-Date: [0-9: +-]+", "POT-Creation-Date: ", output)
+
+    file_regression.check(output, extension=".pot")

tests/test_sphinx/test_sphinx_builds/test_gettext.pot

@@ -0,0 +1,61 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2020, Executable Book Project
+# This file is distributed under the same license as the Python package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: Python \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: \n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: ../../index.md:1
+msgid "**bold** text 1"
+msgstr ""
+
+#: ../../index.md:3
+msgid "**bold** text 2"
+msgstr ""
+
+#: ../../index.md:5
+msgid "**bold** text 3"
+msgstr ""
+
+#: ../../index.md:10
+msgid "**bold** text 4"
+msgstr ""
+
+#: ../../index.md:13
+msgid "**bold** text 5"
+msgstr ""
+
+#: ../../index.md:15
+msgid "**bold** text 6"
+msgstr ""
+
+#: ../../index.md:17
+msgid "**bold** text 7"
+msgstr ""
+
+#: ../../index.md:18
+msgid "**bold** text 8"
+msgstr ""
+
+#: ../../index.md:0
+msgid "**bold** text 9"
+msgstr ""
+
+#: ../../index.md:0
+msgid "**bold** text 10"
+msgstr ""
+
+#: ../../index.md:26
+msgid "**bold** text 11"
+msgstr ""