Add _parse_str_to_doctree() helper method (#13673)

Author: AA-Turner

sphinx/testing/restructuredtext.py

@@ -2,14 +2,12 @@
 
 from typing import TYPE_CHECKING
 
-from docutils import nodes
-
-from sphinx.io import SphinxBaseReader
 from sphinx.parsers import RSTParser
-from sphinx.transforms import SphinxTransformer
-from sphinx.util.docutils import LoggingReporter, _get_settings, sphinx_domains
+from sphinx.util.docutils import _parse_str_to_doctree
 
 if TYPE_CHECKING:
+    from docutils import nodes
+
     from sphinx.application import Sphinx
 
 
@@ -20,49 +18,29 @@ def parse(app: Sphinx, text: str, docname: str = 'index') -> nodes.document:
     registry = app.registry
     srcdir = app.srcdir
 
-    source_path = str(srcdir / f'{docname}.rst')
-
     # Get settings
     settings_overrides = {
+        'env': env,
         'gettext_compact': True,
         'input_encoding': 'utf-8',
         'output_encoding': 'unicode',
         'traceback': True,
     }
-    settings = _get_settings(SphinxBaseReader, RSTParser, defaults=settings_overrides)
-    settings._source = source_path
-    settings.env = env
 
     # Create parser
     parser = RSTParser()
     parser._config = config
     parser._env = env
 
-    # Create root document node
-    reporter = LoggingReporter(
-        source_path,
-        settings.report_level,
-        settings.halt_level,
-        settings.debug,
-        settings.error_encoding_error_handler,
-    )
-    document = nodes.document(settings, reporter, source=source_path)
-    document.note_source(source_path, -1)
-
-    # substitute transformer
-    document.transformer = transformer = SphinxTransformer(document)
-    transformer.add_transforms(SphinxBaseReader().get_transforms())
-    transformer.add_transforms(registry.get_transforms())
-    transformer.add_transforms(parser.get_transforms())
-
     env.current_document.docname = docname
     try:
-        with sphinx_domains(env):
-            parser.parse(text, document)
-            document.current_source = document.current_line = None
-
-            transformer.apply_transforms()
+        return _parse_str_to_doctree(
+            text,
+            filename=srcdir / f'{docname}.rst',
+            default_settings=settings_overrides,
+            env=env,
+            parser=parser,
+            transforms=registry.get_transforms(),
+        )
     finally:
         env.current_document.docname = ''
-
-    return document

sphinx/transforms/__init__.py

@@ -18,7 +18,6 @@
 from sphinx.deprecation import _deprecation_warning
 from sphinx.locale import _, __
 from sphinx.util import logging
-from sphinx.util.docutils import new_document
 from sphinx.util.i18n import format_date
 from sphinx.util.nodes import apply_source_workaround, is_smartquotable
 
@@ -97,6 +96,8 @@ def apply_transforms(self) -> None:
         else:
             # wrap the target node by document node during transforming
             try:
+                from sphinx.util.docutils import new_document
+
                 document = new_document('')
                 if self.env:
                     document.settings.env = self.env

sphinx/util/docutils.py

@@ -5,7 +5,7 @@
 import os
 import re
 import warnings
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
 from copy import copy
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -15,12 +15,15 @@
 from docutils.frontend import OptionParser
 from docutils.io import FileOutput
 from docutils.parsers.rst import Directive, directives, roles
+from docutils.readers import standalone
 from docutils.statemachine import StateMachine
+from docutils.transforms.references import DanglingReferences
 from docutils.utils import Reporter, unescape
 
 from sphinx.errors import SphinxError
 from sphinx.locale import __
-from sphinx.util import logging
+from sphinx.transforms import SphinxTransformer
+from sphinx.util import logging, rst
 from sphinx.util.parsing import nested_parse_to_nodes
 
 logger = logging.getLogger(__name__)
@@ -36,8 +39,10 @@
     from docutils import Component
     from docutils.frontend import Values
     from docutils.nodes import Element, Node, system_message
+    from docutils.parsers import Parser
     from docutils.parsers.rst.states import Inliner
     from docutils.statemachine import State, StringList
+    from docutils.transforms import Transform
 
     from sphinx.builders import Builder
     from sphinx.config import Config
@@ -69,6 +74,13 @@ def __call__(
         ) -> tuple[RoleFunction | None, list[system_message]]: ...
 
 
+_READER_TRANSFORMS = [
+    transform
+    for transform in standalone.Reader().get_transforms()
+    if transform is not DanglingReferences
+]
+
+
 additional_nodes: set[type[Element]] = set()
 
 
@@ -821,6 +833,55 @@ def new_document(source_path: str, settings: Any = None) -> nodes.document:
     return document
 
 
+def _parse_str_to_doctree(
+    content: str,
+    *,
+    filename: Path,
+    default_role: str = '',
+    default_settings: Mapping[str, Any],
+    env: BuildEnvironment,
+    parser: Parser,
+    transforms: Sequence[type[Transform]] = (),
+) -> nodes.document:
+    env.current_document._parser = parser
+
+    # Propagate exceptions by default when used programmatically:
+    defaults = {'traceback': True, **default_settings}
+    settings = _get_settings(standalone.Reader, parser, defaults=defaults)
+    settings._source = str(filename)
+
+    # Create root document node
+    reporter = LoggingReporter(
+        source=str(filename),
+        report_level=settings.report_level,
+        halt_level=settings.halt_level,
+        debug=settings.debug,
+        error_handler=settings.error_encoding_error_handler,
+    )
+    document = nodes.document(settings, reporter, source=str(filename))
+    document.note_source(str(filename), -1)
+
+    # substitute transformer
+    document.transformer = transformer = SphinxTransformer(document)
+    transformer.add_transforms(_READER_TRANSFORMS)
+    transformer.add_transforms(transforms)
+    transformer.add_transforms(parser.get_transforms())
+
+    if default_role:
+        default_role_cm = rst.default_role(env.current_document.docname, default_role)
+    else:
+        default_role_cm = nullcontext()  # type: ignore[assignment]
+    with sphinx_domains(env), default_role_cm:
+        # parse content to abstract syntax tree
+        parser.parse(content, document)
+        document.current_source = document.current_line = None
+
+        # run transforms
+        transformer.apply_transforms()
+
+    return document
+
+
 def _get_settings(
     *components: Component | type[Component],
     defaults: Mapping[str, Any],

tests/test_directives/test_directive_object_description.py

@@ -9,9 +9,8 @@
 from docutils import nodes
 
 from sphinx import addnodes
-from sphinx.io import _create_publisher
 from sphinx.testing import restructuredtext
-from sphinx.util.docutils import sphinx_domains
+from sphinx.util.docutils import _parse_str_to_doctree
 
 if TYPE_CHECKING:
     from sphinx.application import Sphinx
@@ -24,15 +23,20 @@ def _doctree_for_test(
 ) -> nodes.document:
     config = app.config
     registry = app.registry
+
+    filename = env.doc2path(docname)
+    content = filename.read_text(encoding='utf-8')
+
     env.prepare_settings(docname)
     parser = registry.create_source_parser('restructuredtext', config=config, env=env)
-    publisher = _create_publisher(
-        env=env, parser=parser, transforms=registry.get_transforms()
+    return _parse_str_to_doctree(
+        content,
+        filename=env.doc2path(docname),
+        default_settings={'env': env},
+        env=env,
+        parser=parser,
+        transforms=registry.get_transforms(),
     )
-    with sphinx_domains(env):
-        publisher.set_source(source_path=str(env.doc2path(docname)))
-        publisher.publish()
-        return publisher.document
 
 
 @pytest.mark.sphinx('text', testroot='object-description-sections')