test that matplotlib event loop integration is responsive (#1463)

Author: minrk

ipykernel/eventloops.py

@@ -373,7 +373,7 @@ def loop_gtk3_exit(kernel):
     kernel._gtk.stop()
 
 
-@register_integration("osx")
+@register_integration("osx", "macosx")
 def loop_cocoa(kernel):
     """Start the kernel, coordinating with the Cocoa CFRunLoop event loop
     via the matplotlib MacOSX backend.

pyproject.toml

@@ -57,6 +57,7 @@ docs = [
 test = [
     "pytest>=7.0,<9",
     "pytest-cov",
+    # 'pytest-xvfb; platform_system == "Linux"',
     "flaky",
     "ipyparallel",
     "pre-commit",

tests/conftest.py

@@ -4,6 +4,7 @@
 from typing import no_type_check
 from unittest.mock import MagicMock
 
+import pytest
 import pytest_asyncio
 import zmq
 from jupyter_client.session import Session
@@ -20,6 +21,7 @@
     # Windows
     resource = None  # type:ignore
 
+from .utils import new_kernel
 
 # Handle resource limit
 # Ensure a minimal soft limit of DEFAULT_SOFT if the current hard limit is at least that much.
@@ -158,3 +160,9 @@ def ipkernel():
     yield kernel
     kernel.destroy()
     ZMQInteractiveShell.clear_instance()
+
+
+@pytest.fixture
+def kc():
+    with new_kernel() as kc:
+        yield kc

tests/test_eventloop.py

@@ -56,6 +56,11 @@ def _setup_env():
 
 windows_skip = pytest.mark.skipif(os.name == "nt", reason="causing failures on windows")
 
+# some part of this module seems to hang when run with xvfb
+pytestmark = pytest.mark.skipif(
+    sys.platform == "linux" and bool(os.getenv("CI")), reason="hangs on linux CI"
+)
+
 
 @windows_skip
 @pytest.mark.skipif(sys.platform == "darwin", reason="hangs on macos")

tests/test_matplotlib_eventloops.py

@@ -0,0 +1,106 @@
+import os
+import sys
+import time
+
+import pytest
+from jupyter_client.blocking.client import BlockingKernelClient
+
+from .test_eventloop import qt_guis_avail
+from .utils import assemble_output
+
+# these tests don't seem to work with xvfb yet
+# these tests seem to be a problem on CI in general
+pytestmark = pytest.mark.skipif(
+    bool(os.getenv("CI")),
+    reason="tests not working yet reliably on CI",
+)
+
+guis = []
+if not sys.platform.startswith("tk"):
+    guis.append("tk")
+if qt_guis_avail:
+    guis.append("qt")
+if sys.platform == "darwin":
+    guis.append("osx")
+
+backends = {
+    "tk": "tkagg",
+    "qt": "qtagg",
+    "osx": "macosx",
+}
+
+
+def execute(
+    kc: BlockingKernelClient,
+    code: str,
+    timeout=120,
+):
+    msg_id = kc.execute(code)
+    stdout, stderr = assemble_output(kc.get_iopub_msg, timeout=timeout, parent_msg_id=msg_id)
+    assert not stderr.strip()
+    return stdout.strip(), stderr.strip()
+
+
+@pytest.mark.parametrize("gui", guis)
+@pytest.mark.timeout(300)
+def test_matplotlib_gui(kc, gui):
+    """Make sure matplotlib activates and its eventloop runs while the kernel is also responsive"""
+    pytest.importorskip("matplotlib", reason="this test requires matplotlib")
+    stdout, stderr = execute(kc, f"%matplotlib {gui}")
+    assert not stderr
+    # debug: show output from invoking the matplotlib magic
+    print(stdout)
+    execute(
+        kc,
+        """
+    from concurrent.futures import Future
+    import matplotlib as mpl
+    import matplotlib.pyplot as plt
+    """,
+    )
+    stdout, _ = execute(kc, "print(mpl.get_backend())")
+    assert stdout == backends[gui]
+    execute(
+        kc,
+        """
+fig, ax = plt.subplots()
+timer = fig.canvas.new_timer(interval=10)
+f = Future()
+
+call_count = 0
+def add_call():
+    global call_count
+    call_count += 1
+    if not f.done():
+        f.set_result(None)
+
+timer.add_callback(add_call)
+timer.start()
+""",
+    )
+    # wait for the first call (up to 60 seconds)
+    deadline = time.monotonic() + 60
+    done = False
+    while time.monotonic() <= deadline:
+        stdout, _ = execute(kc, "print(f.done())")
+        if stdout.strip() == "True":
+            done = True
+            break
+        if stdout == "False":
+            time.sleep(0.1)
+        else:
+            pytest.fail(f"Unexpected output {stdout}")
+    if not done:
+        pytest.fail("future never finished...")
+
+    time.sleep(0.25)
+    stdout, _ = execute(kc, "print(call_count)")
+    call_count = int(stdout)
+    assert call_count > 0
+    time.sleep(0.25)
+    stdout, _ = execute(kc, "timer.stop()\nprint(call_count)")
+    call_count_2 = int(stdout)
+    assert call_count_2 > call_count
+    stdout, _ = execute(kc, "print(call_count)")
+    call_count_3 = int(stdout)
+    assert call_count_3 <= call_count_2 + 5

tests/utils.py

@@ -168,7 +168,7 @@ def new_kernel(argv=None):
     return manager.run_kernel(**kwargs)
 
 
-def assemble_output(get_msg, timeout=1, parent_msg_id: str | None = None):
+def assemble_output(get_msg, timeout=1, parent_msg_id: str | None = None, raise_error=True):
     """assemble stdout/err from an execution"""
     stdout = ""
     stderr = ""
@@ -191,6 +191,12 @@ def assemble_output(get_msg, timeout=1, parent_msg_id: str | None = None):
                 stderr += content["text"]
             else:
                 raise KeyError("bad stream: %r" % content["name"])
+        elif raise_error and msg["msg_type"] == "error":
+            tb = "\n".join(msg["content"]["traceback"])
+            msg = f"Execution failed with:\n{tb}"
+            if stderr:
+                msg = f"{msg}\nstderr:\n{stderr}"
+            raise RuntimeError(msg)
         else:
             # other output, ignored
             pass