Add doctest_fail_fast option to exit after the first failed test (#13332)

Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com>

Author: tillahoffmann

AUTHORS.rst

@@ -106,6 +106,7 @@ Contributors
 * Taku Shimizu -- epub3 builder
 * Thomas Lamb -- linkcheck builder
 * Thomas Waldmann -- apidoc module fixes
+* Till Hoffmann -- doctest option to exit after first failed test
 * Tim Hoffmann -- theme improvements
 * Victor Wheeler -- documentation improvements
 * Vince Salvino -- JavaScript search improvements

CHANGES.rst

@@ -13,6 +13,10 @@ Deprecated
 Features added
 --------------
 
+* #13332: Add :confval:`doctest_fail_fast` option to exit after the first failed
+  test.
+  Patch by Till Hoffmann.
+
 Bugs fixed
 ----------
 

doc/usage/extensions/doctest.rst

@@ -452,3 +452,11 @@ The doctest extension uses the following configuration values:
    Also, removal of ``<BLANKLINE>`` and ``# doctest:`` options only works in
    :rst:dir:`doctest` blocks, though you may set :confval:`trim_doctest_flags`
    to achieve that in all code blocks with Python console content.
+
+.. confval:: doctest_fail_fast
+   :type: :code-py:`bool`
+   :default: :code-py:`False`
+
+   Exit when the first failure is encountered.
+
+   .. versionadded:: 8.3

sphinx/ext/doctest.py

@@ -358,10 +358,17 @@ def finish(self) -> None:
         def s(v: int) -> str:
             return 's' if v != 1 else ''
 
+        header = 'Doctest summary'
+        if self.total_failures or self.setup_failures or self.cleanup_failures:
+            self.app.statuscode = 1
+            if self.config.doctest_fail_fast:
+                header = f'{header} (exiting after first failed test)'
+        underline = '=' * len(header)
+
         self._out(
             f"""
-Doctest summary
-===============
+{header}
+{underline}
 {self.total_tries:5} test{s(self.total_tries)}
 {self.total_failures:5} failure{s(self.total_failures)} in tests
 {self.setup_failures:5} failure{s(self.setup_failures)} in setup code
@@ -370,15 +377,14 @@ def s(v: int) -> str:
         )
         self.outfile.close()
 
-        if self.total_failures or self.setup_failures or self.cleanup_failures:
-            self.app.statuscode = 1
-
     def write_documents(self, docnames: Set[str]) -> None:
         logger.info(bold('running tests...'))
         for docname in sorted(docnames):
             # no need to resolve the doctree
             doctree = self.env.get_doctree(docname)
-            self.test_doc(docname, doctree)
+            success = self.test_doc(docname, doctree)
+            if not success and self.config.doctest_fail_fast:
+                break
 
     def get_filename_for_node(self, node: Node, docname: str) -> str:
         """Try to get the file which actually contains the doctest, not the
@@ -419,7 +425,7 @@ def skipped(self, node: Element) -> bool:
                 exec(self.config.doctest_global_cleanup, context)  # NoQA: S102
             return should_skip
 
-    def test_doc(self, docname: str, doctree: Node) -> None:
+    def test_doc(self, docname: str, doctree: Node) -> bool:
         groups: dict[str, TestGroup] = {}
         add_to_all_groups = []
         self.setup_runner = SphinxDocTestRunner(verbose=False, optionflags=self.opt)
@@ -496,13 +502,17 @@ def condition(node: Node) -> bool:
             for group in groups.values():
                 group.add_code(code)
         if not groups:
-            return
+            return True
 
         show_successes = self.config.doctest_show_successes
         if show_successes:
             self._out(f'\nDocument: {docname}\n----------{"-" * len(docname)}\n')
+        success = True
         for group in groups.values():
-            self.test_group(group)
+            if not self.test_group(group):
+                success = False
+                if self.config.doctest_fail_fast:
+                    break
         # Separately count results from setup code
         res_f, res_t = self.setup_runner.summarize(self._out, verbose=False)
         self.setup_failures += res_f
@@ -517,13 +527,14 @@ def condition(node: Node) -> bool:
             )
             self.cleanup_failures += res_f
             self.cleanup_tries += res_t
+        return success
 
     def compile(
         self, code: str, name: str, type: str, flags: Any, dont_inherit: bool
     ) -> Any:
         return compile(code, name, self.type, flags, dont_inherit)
 
-    def test_group(self, group: TestGroup) -> None:
+    def test_group(self, group: TestGroup) -> bool:
         ns: dict[str, Any] = {}
 
         def run_setup_cleanup(
@@ -553,9 +564,10 @@ def run_setup_cleanup(
         # run the setup code
         if not run_setup_cleanup(self.setup_runner, group.setup, 'setup'):
             # if setup failed, don't run the group
-            return
+            return False
 
         # run the tests
+        success = True
         for code in group.tests:
             if len(code) == 1:
                 # ordinary doctests (code/output interleaved)
@@ -608,11 +620,19 @@ def run_setup_cleanup(
                 self.type = 'exec'  # multiple statements again
             # DocTest.__init__ copies the globs namespace, which we don't want
             test.globs = ns
+            old_f = self.test_runner.failures
             # also don't clear the globs namespace after running the doctest
             self.test_runner.run(test, out=self._warn_out, clear_globs=False)
+            if self.test_runner.failures > old_f:
+                success = False
+                if self.config.doctest_fail_fast:
+                    break
 
         # run the cleanup
-        run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup')
+        if not run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup'):
+            return False
+
+        return success
 
 
 def setup(app: Sphinx) -> ExtensionMetadata:
@@ -638,6 +658,7 @@ def setup(app: Sphinx) -> ExtensionMetadata:
         '',
         types=frozenset({int}),
     )
+    app.add_config_value('doctest_fail_fast', False, '', types=frozenset({bool}))
     return {
         'version': sphinx.__display_version__,
         'parallel_read_safe': True,

tests/roots/test-ext-doctest-fail-fast/conf.py

@@ -0,0 +1,11 @@
+extensions = ['sphinx.ext.doctest']
+
+project = 'test project for doctest'
+root_doc = 'fail-fast'
+source_suffix = {
+    '.txt': 'restructuredtext',
+}
+exclude_patterns = ['_build']
+
+# Set in tests.
+# doctest_fail_fast = ...

tests/roots/test-ext-doctest-fail-fast/fail-fast.txt

@@ -0,0 +1,11 @@
+Testing fast failure in the doctest extension
+=============================================
+
+>>> 1 + 1
+2
+
+>>> 1 + 1
+3
+
+>>> 1 + 1
+3

tests/test_extensions/test_ext_doctest.py

@@ -147,3 +147,23 @@ def test_reporting_with_autodoc(app, capfd):
     assert 'File "dir/bar.py", line ?, in default' in failures
     assert 'File "foo.py", line ?, in default' in failures
     assert 'File "index.rst", line 4, in default' in failures
+
+
+@pytest.mark.sphinx('doctest', testroot='ext-doctest-fail-fast')
+@pytest.mark.parametrize('fail_fast', [False, True, None])
+def test_fail_fast(app, fail_fast, capsys):
+    if fail_fast is not None:
+        app.config.doctest_fail_fast = fail_fast
+    # Patch builder to get a copy of the output
+    written = []
+    app.builder._out = written.append
+    app.build(force_all=True)
+    assert app.statuscode
+
+    written = ''.join(written)
+    if fail_fast:
+        assert 'Doctest summary (exiting after first failed test)' in written
+        assert '1 failure in tests' in written
+    else:
+        assert 'Doctest summary\n' in written
+        assert '2 failures in tests' in written