Use Docutils translators directly in the writing phase (#13683)

Author: AA-Turner

sphinx/builders/html/__init__.py

@@ -15,12 +15,11 @@
 from typing import TYPE_CHECKING
 from urllib.parse import quote
 
+import docutils.parsers.rst
 import docutils.readers.doctree
 import docutils.utils
 import jinja2.exceptions
 from docutils import nodes
-from docutils.core import Publisher
-from docutils.io import DocTreeInput, StringOutput
 
 from sphinx import __display_version__, package_dir
 from sphinx import version_info as sphinx_version
@@ -69,7 +68,6 @@
     from typing import Any, TypeAlias
 
     from docutils.nodes import Node
-    from docutils.readers import Reader
 
     from sphinx.application import Sphinx
     from sphinx.config import Config
@@ -93,6 +91,10 @@
     bool,
 ]
 
+_READER_TRANSFORMS = docutils.readers.doctree.Reader().get_transforms()
+_PARSER_TRANSFORMS = docutils.parsers.rst.Parser().get_transforms()
+_WRITER_TRANSFORMS = HTMLWriter(None).get_transforms()  # type: ignore[arg-type]
+
 
 def convert_locale_to_language_tag(locale: str | None) -> str | None:
     """Convert a locale string to a language tag (ex. en_US -> en-US).
@@ -150,19 +152,13 @@ def __init__(self, app: Sphinx, env: BuildEnvironment) -> None:
         # JS files
         self._js_files: list[_JavaScript] = []
 
-        # Cached Publisher for writing doctrees to HTML
-        reader: Reader[DocTreeInput] = docutils.readers.doctree.Reader(
-            parser_name='restructuredtext'
-        )
-        pub = Publisher(
-            reader=reader,
-            parser=reader.parser,
-            writer=HTMLWriter(self),
-            source_class=DocTreeInput,
-            destination=StringOutput(encoding='unicode'),
+        # Cached settings for render_partial()
+        self._settings = _get_settings(
+            docutils.readers.doctree.Reader,
+            docutils.parsers.rst.Parser,
+            HTMLWriter,
+            defaults={'output_encoding': 'unicode', 'traceback': True},
         )
-        pub.get_settings(output_encoding='unicode', traceback=True)
-        self._publisher = pub
 
     def init(self) -> None:
         self.build_info = self.create_build_info()
@@ -428,10 +424,11 @@ def render_partial(self, node: Node | None) -> dict[str, str]:
         """Utility: Render a lone doctree node."""
         if node is None:
             return {'fragment': ''}
-        pub = self._publisher
-        doc = docutils.utils.new_document('<partial node>', pub.settings)
+        doc = docutils.utils.new_document('<partial node>', self._settings)
         doc.append(node)
-        doc.transformer.populate_from_components((pub.reader, pub.parser, pub.writer))
+        doc.transformer.add_transforms(_READER_TRANSFORMS)
+        doc.transformer.add_transforms(_PARSER_TRANSFORMS)
+        doc.transformer.add_transforms(_WRITER_TRANSFORMS)
         doc.transformer.apply_transforms()
         visitor: HTML5Translator = self.create_translator(doc, self)  # type: ignore[assignment]
         doc.walkabout(visitor)
@@ -456,7 +453,6 @@ def prepare_writing(self, docnames: Set[str]) -> None:
             )
             self.load_indexer(docnames)
 
-        self.docwriter = HTMLWriter(self)
         self.docsettings = _get_settings(
             HTMLWriter, defaults=self.env.settings, read_config_files=True
         )
@@ -666,21 +662,20 @@ def copy_assets(self) -> None:
         self.finish_tasks.join()
 
     def write_doc(self, docname: str, doctree: nodes.document) -> None:
-        destination = StringOutput(encoding='utf-8')
         doctree.settings = self.docsettings
 
         self.secnumbers = self.env.toc_secnumbers.get(docname, {})
         self.fignumbers = self.env.toc_fignumbers.get(docname, {})
         self.imgpath = relative_uri(self.get_target_uri(docname), '_images')
         self.dlpath = relative_uri(self.get_target_uri(docname), '_downloads')
         self.current_docname = docname
-        self.docwriter.write(doctree, destination)
-        self.docwriter.assemble_parts()
-        body = self.docwriter.parts['fragment']
-        metatags = self.docwriter.clean_meta
+        visitor: HTML5Translator = self.create_translator(doctree, self)  # type: ignore[assignment]
+        doctree.walkabout(visitor)
+        body = ''.join(visitor.fragment)
+        clean_meta = ''.join(visitor.meta[2:])
 
-        ctx = self.get_doc_context(docname, body, metatags)
-        ctx['has_maths_elements'] = self.docwriter._has_maths_elements
+        ctx = self.get_doc_context(docname, body, clean_meta)
+        ctx['has_maths_elements'] = getattr(visitor, '_has_maths_elements', False)
         self.handle_page(docname, ctx, event_arg=doctree)
 
     def write_doc_serialized(self, docname: str, doctree: nodes.document) -> None:

sphinx/builders/latex/__init__.py

@@ -24,7 +24,7 @@
 from sphinx.locale import _, __
 from sphinx.util import logging, texescape
 from sphinx.util.display import progress_message, status_iterator
-from sphinx.util.docutils import SphinxFileOutput, _get_settings, new_document
+from sphinx.util.docutils import _get_settings, new_document
 from sphinx.util.fileutil import copy_asset_file
 from sphinx.util.i18n import format_date
 from sphinx.util.nodes import inline_all_toctrees
@@ -297,7 +297,6 @@ def copy_assets(self) -> None:
             self.copy_latex_additional_files()
 
     def write_documents(self, _docnames: Set[str]) -> None:
-        docwriter = LaTeXWriter(self)
         docsettings = _get_settings(
             LaTeXWriter, defaults=self.env.settings, read_config_files=True
         )
@@ -308,11 +307,6 @@ def write_documents(self, _docnames: Set[str]) -> None:
             toctree_only = False
             if len(entry) > 5:
                 toctree_only = entry[5]
-            destination = SphinxFileOutput(
-                destination_path=self.outdir / targetname,
-                encoding='utf-8',
-                overwrite_if_changed=True,
-            )
             with progress_message(__('processing %s') % targetname, nonl=False):
                 doctree = self.env.get_doctree(docname)
                 toctree = next(doctree.findall(addnodes.toctree), None)
@@ -343,8 +337,16 @@ def write_documents(self, _docnames: Set[str]) -> None:
                 docsettings._docclass = theme.name
 
                 doctree.settings = docsettings
-                docwriter.theme = theme
-                docwriter.write(doctree, destination)
+                visitor: LaTeXTranslator = self.create_translator(doctree, self, theme)  # type: ignore[assignment]
+                doctree.walkabout(visitor)
+                output = visitor.astext()
+                destination_path = self.outdir / targetname
+                # https://github.com/sphinx-doc/sphinx/issues/4362
+                if (
+                    not destination_path.is_file()
+                    or destination_path.read_bytes() != output.encode()
+                ):
+                    destination_path.write_text(output, encoding='utf-8')
 
     def get_contentsname(self, indexfile: str) -> str:
         tree = self.env.get_doctree(indexfile)

sphinx/builders/manpage.py

@@ -4,8 +4,6 @@
 
 from typing import TYPE_CHECKING
 
-from docutils.io import FileOutput
-
 from sphinx import addnodes
 from sphinx._cli.util.colour import darkgreen
 from sphinx.builders import Builder
@@ -15,7 +13,11 @@
 from sphinx.util.docutils import _get_settings
 from sphinx.util.nodes import inline_all_toctrees
 from sphinx.util.osutil import ensuredir, make_filename_from_project
-from sphinx.writers.manpage import ManualPageTranslator, ManualPageWriter
+from sphinx.writers.manpage import (
+    ManualPageTranslator,
+    ManualPageWriter,
+    NestedInlineTransform,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Set
@@ -51,7 +53,6 @@ def get_target_uri(self, docname: str, typ: str | None = None) -> str:
 
     @progress_message(__('writing'))
     def write_documents(self, _docnames: Set[str]) -> None:
-        docwriter = ManualPageWriter(self)
         docsettings = _get_settings(
             ManualPageWriter, defaults=self.env.settings, read_config_files=True
         )
@@ -83,10 +84,6 @@ def write_documents(self, _docnames: Set[str]) -> None:
                 targetname = f'{name}.{section}'
 
             logger.info('%s { ', darkgreen(targetname))
-            destination = FileOutput(
-                destination_path=self.outdir / targetname,
-                encoding='utf-8',
-            )
 
             tree = self.env.get_doctree(docname)
             docnames: set[str] = set()
@@ -100,7 +97,11 @@ def write_documents(self, _docnames: Set[str]) -> None:
             for pendingnode in largetree.findall(addnodes.pending_xref):
                 pendingnode.replace_self(pendingnode.children)
 
-            docwriter.write(largetree, destination)
+            transform = NestedInlineTransform(largetree)
+            transform.apply()
+            visitor: ManualPageTranslator = self.create_translator(largetree, self)  # type: ignore[assignment]
+            largetree.walkabout(visitor)
+            (self.outdir / targetname).write_text(visitor.astext(), encoding='utf-8')
 
     def finish(self) -> None:
         pass

sphinx/builders/texinfo.py

@@ -6,7 +6,6 @@
 from typing import TYPE_CHECKING
 
 from docutils import nodes
-from docutils.io import FileOutput
 
 from sphinx import addnodes, package_dir
 from sphinx._cli.util.colour import darkgreen
@@ -103,10 +102,6 @@ def write_documents(self, _docnames: Set[str]) -> None:
             toctree_only = False
             if len(entry) > 7:
                 toctree_only = entry[7]
-            destination = FileOutput(
-                destination_path=self.outdir / targetname,
-                encoding='utf-8',
-            )
             with progress_message(__('processing %s') % targetname, nonl=False):
                 appendices = self.config.texinfo_appendices or []
                 doctree = self.assemble_doctree(
@@ -115,7 +110,6 @@ def write_documents(self, _docnames: Set[str]) -> None:
 
             with progress_message(__('writing')):
                 self.post_process_images(doctree)
-                docwriter = TexinfoWriter(self)
                 settings = _get_settings(
                     TexinfoWriter, defaults=self.env.settings, read_config_files=True
                 )
@@ -128,7 +122,10 @@ def write_documents(self, _docnames: Set[str]) -> None:
                 settings.texinfo_dir_description = description or ''
                 settings.docname = docname
                 doctree.settings = settings
-                docwriter.write(doctree, destination)
+                visitor: TexinfoTranslator = self.create_translator(doctree, self)  # type: ignore[assignment]
+                doctree.walkabout(visitor)
+                visitor.finish()
+                (self.outdir / targetname).write_text(visitor.output, encoding='utf-8')
                 self.copy_image_files(targetname[:-5])
 
     def assemble_doctree(

sphinx/builders/text.py

@@ -4,16 +4,14 @@
 
 from typing import TYPE_CHECKING
 
-from docutils.io import StringOutput
-
 from sphinx.builders import Builder
 from sphinx.locale import __
 from sphinx.util import logging
 from sphinx.util.osutil import _last_modified_time
-from sphinx.writers.text import TextTranslator, TextWriter
+from sphinx.writers.text import TextTranslator
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator, Set
+    from collections.abc import Iterator
 
     from docutils import nodes
 
@@ -59,19 +57,16 @@ def get_outdated_docs(self) -> Iterator[str]:
     def get_target_uri(self, docname: str, typ: str | None = None) -> str:
         return ''
 
-    def prepare_writing(self, docnames: Set[str]) -> None:
-        self.writer = TextWriter(self)
-
     def write_doc(self, docname: str, doctree: nodes.document) -> None:
         self.current_docname = docname
         self.secnumbers = self.env.toc_secnumbers.get(docname, {})
-        destination = StringOutput(encoding='utf-8')
-        self.writer.write(doctree, destination)
+        visitor: TextTranslator = self.create_translator(doctree, self)  # type: ignore[assignment]
+        doctree.walkabout(visitor)
+        output = visitor.body
         out_file_name = self.outdir / (docname + self.out_suffix)
         out_file_name.parent.mkdir(parents=True, exist_ok=True)
         try:
-            with open(out_file_name, 'w', encoding='utf-8') as f:
-                f.write(self.writer.output)
+            out_file_name.write_text(output, encoding='utf-8')
         except OSError as err:
             logger.warning(__('error writing file %s: %s'), out_file_name, err)
 

sphinx/builders/xml.py

@@ -5,17 +5,15 @@
 from typing import TYPE_CHECKING
 
 from docutils import nodes
-from docutils.io import StringOutput
 from docutils.writers.docutils_xml import XMLTranslator
 
 from sphinx.builders import Builder
 from sphinx.locale import __
 from sphinx.util import logging
 from sphinx.util.osutil import _last_modified_time
-from sphinx.writers.xml import PseudoXMLWriter, XMLWriter
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator, Set
+    from collections.abc import Iterator
 
     from sphinx.application import Sphinx
     from sphinx.util.typing import ExtensionMetadata
@@ -33,8 +31,6 @@ class XMLBuilder(Builder):
     out_suffix = '.xml'
     allow_parallel = True
 
-    _writer_class: type[XMLWriter | PseudoXMLWriter] = XMLWriter
-    writer: XMLWriter | PseudoXMLWriter
     default_translator_class = XMLTranslator
 
     def init(self) -> None:
@@ -61,9 +57,6 @@ def get_outdated_docs(self) -> Iterator[str]:
     def get_target_uri(self, docname: str, typ: str | None = None) -> str:
         return docname
 
-    def prepare_writing(self, docnames: Set[str]) -> None:
-        self.writer = self._writer_class(self)
-
     def write_doc(self, docname: str, doctree: nodes.document) -> None:
         # work around multiple string % tuple issues in docutils;
         # replace tuples in attribute values with lists
@@ -79,16 +72,25 @@ def write_doc(self, docname: str, doctree: nodes.document) -> None:
                     for i, val in enumerate(value):
                         if isinstance(val, tuple):
                             value[i] = list(val)
-        destination = StringOutput(encoding='utf-8')
-        self.writer.write(doctree, destination)
+        output = self._translate(doctree)
         out_file_name = self.outdir / (docname + self.out_suffix)
         out_file_name.parent.mkdir(parents=True, exist_ok=True)
         try:
-            with open(out_file_name, 'w', encoding='utf-8') as f:
-                f.write(self.writer.output)
+            out_file_name.write_text(output, encoding='utf-8')
         except OSError as err:
             logger.warning(__('error writing file %s: %s'), out_file_name, err)
 
+    def _translate(self, doctree: nodes.document) -> str:
+        doctree.settings.newlines = doctree.settings.indents = self.config.xml_pretty
+        doctree.settings.xml_declaration = True
+        doctree.settings.doctype_declaration = True
+
+        # copied from docutils.writers.docutils_xml.Writer.translate()
+        # so that we can override the translator class
+        visitor: XMLTranslator = self.create_translator(doctree)
+        doctree.walkabout(visitor)
+        return ''.join(visitor.output)
+
     def finish(self) -> None:
         pass
 
@@ -102,7 +104,8 @@ class PseudoXMLBuilder(XMLBuilder):
 
     out_suffix = '.pseudoxml'
 
-    _writer_class = PseudoXMLWriter
+    def _translate(self, doctree: nodes.document) -> str:
+        return doctree.pformat()
 
 
 def setup(app: Sphinx) -> ExtensionMetadata:

sphinx/parsers.py

@@ -70,7 +70,7 @@ def get_transforms(self) -> list[type[Transform]]:
 
         refs: sphinx.io.SphinxStandaloneReader
         """
-        transforms = super().get_transforms()
+        transforms = super(RSTParser, RSTParser()).get_transforms()
         transforms.remove(SmartQuotes)
         return transforms
 

sphinx/writers/html.py

@@ -4,7 +4,7 @@
 
 from typing import TYPE_CHECKING, cast
 
-from docutils.writers.html4css1 import Writer
+from docutils.writers import html4css1
 
 from sphinx.util import logging
 from sphinx.writers.html5 import HTML5Translator
@@ -20,7 +20,7 @@
 # https://www.arnebrodowski.de/blog/write-your-own-restructuredtext-writer.html
 
 
-class HTMLWriter(Writer):  # type: ignore[misc]
+class HTMLWriter(html4css1.Writer):  # type: ignore[misc]
     # override embed-stylesheet default value to False.
     settings_default_overrides = {'embed_stylesheet': False}