gettext: Ignore trailing backslashes when extracting messages (#13686)

Trailing backslashes are now ignored by the `gettext` builder,
and whitespaces preceding and following them are ignored.

Author: picnixz

CHANGES.rst

@@ -77,6 +77,8 @@ Bugs fixed
 * #13635: LaTeX: if a cell contains a table, row coloring is turned off for
   the next table cells.
   Patch by Jean-François B.
+* #13685: gettext: Correctly ignore trailing backslashes.
+  Patch by Bénédikt Tran.
 
 Testing
 -------

sphinx/util/nodes.py

@@ -5,6 +5,7 @@
 import contextlib
 import re
 import unicodedata
+from io import StringIO
 from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast
 
 from docutils import nodes
@@ -289,6 +290,35 @@ def is_translatable(node: Node) -> bool:
 )  # fmt: skip
 
 
+def _clean_extracted_message(text: str) -> str:
+    """Remove trailing backslashes from each line of *text*."""
+    if '\\' in text:
+        # TODO(picnixz): if possible, find a regex alternative
+        # that is not vulnerable to a ReDOS (the code below is
+        # equivalent to re.sub(r'[ \t]*\\[ \t]*$', text, re.MULTILINE)).
+        buffer = StringIO()
+        for line in text.splitlines(keepends=True):
+            split = line.rsplit('\\', maxsplit=1)
+            if len(split) == 2:
+                prefix, suffix = split
+                if re.match(r'^[ \t]*\s$', suffix):
+                    # The line ends with some NL character, preceded by
+                    # one or more whitespaces (to be dropped), the backslash,
+                    # and possibly other whitespaces on its left.
+                    buffer.write(prefix.rstrip(' \t'))
+                    buffer.write(suffix.lstrip(' \t'))
+                elif not suffix:
+                    # backslash is at the end of the LAST line
+                    buffer.write(prefix.rstrip(' \t'))
+                else:
+                    # backslash is is in the middle of the line
+                    buffer.write(line)
+            else:
+                buffer.write(line)
+        text = buffer.getvalue()
+    return text.replace('\n', ' ').strip()
+
+
 def extract_messages(doctree: Element) -> Iterable[tuple[Element, str]]:
     """Extract translatable messages from a document tree."""
     for node in doctree.findall(is_translatable):
@@ -311,7 +341,8 @@ def extract_messages(doctree: Element) -> Iterable[tuple[Element, str]]:
         elif isinstance(node, nodes.meta):
             msg = node['content']
         else:
-            msg = node.rawsource.replace('\n', ' ').strip()  # type: ignore[attr-defined]
+            text = node.rawsource  # type: ignore[attr-defined]
+            msg = _clean_extracted_message(text)
 
         # XXX nodes rendering empty are likely a bug in sphinx.addnodes
         if msg:

tests/roots/test-intl/backslashes.txt

@@ -0,0 +1,38 @@
+:tocdepth: 2
+
+i18n with backslashes
+=====================
+
+line 1\
+line 2 \
+line 3  \
+line 4a \ and 4b \
+line       with spaces after backslash     \    
+last line       with spaces     \
+and done 1
+
+.. gettext parses the following lines as "a<space>b<space>c",
+   while a C pre-processor would have produced "a<space>bc".
+
+a \
+b\
+c   \
+
+last trailing \ \ \
+is ignored
+
+
+See [#]_
+
+.. [#] footnote with backslashes \
+   and done 2
+
+
+.. note:: directive with \
+   backslashes
+
+
+.. function:: foo(a,    \
+                  b, \
+                  c, d, e, f)
+   the foo

tests/roots/test-intl/index.txt

@@ -32,6 +32,7 @@ CONTENTS
    translation_progress
    topic
    markup
+   backslashes
 
 .. toctree::
    :maxdepth: 2

tests/test_builders/test_build_gettext.py

@@ -323,3 +323,29 @@ def test_gettext_literalblock_additional(app: SphinxTestApp) -> None:
         "stdout object\\n>>>\\n>>> if __name__ == '__main__':  # if run this py "
         'file as python script\\n...     main()  # call main',
     ]
+
+
+@pytest.mark.sphinx('gettext', testroot='intl', srcdir='gettext')
+def test_gettext_trailing_backslashes(app: SphinxTestApp) -> None:
+    app.build(force_all=True)
+
+    assert (app.outdir / 'backslashes.pot').is_file()
+    pot = (app.outdir / 'backslashes.pot').read_text(encoding='utf8')
+    msg_ids = get_msgids(pot)
+    assert msg_ids == [
+        'i18n with backslashes',
+        (
+            'line 1 line 2 line 3 '
+            # middle backslashes are escaped normally
+            'line 4a \\\\ and 4b '
+            # whitespaces after backslashes are dropped
+            'line       with spaces after backslash '
+            'last line       with spaces '
+            'and done 1'
+        ),
+        'a b c',
+        'last trailing \\\\ \\\\ is ignored',
+        'See [#]_',
+        'footnote with backslashes and done 2',
+        'directive with backslashes',
+    ]