Add flaky pytest infrastructure and weekend runners

.github/workflows/check-weekly-flaky.yaml

@@ -0,0 +1,28 @@
+name: Run Catalyst pytests for weekly flaky test checking
+
+permissions:
+  contents: read
+
+on:
+  schedule:
+    # Run every weekend on Saturday at 23:40 EDT (cron is in UTC)
+    # https://crontab.cronhub.io/
+    - cron: "40 23 * * SAT"
+
+jobs:
+  weekly-flaky-test-run:
+    name: Run weekly flaky tests
+    runs-on: ubuntu-24.04
+
+    steps:
+    - name: Checkout Catalyst repo
+      uses: actions/checkout@v4
+
+    - name: Install Deps
+      run: |
+        python3 -m pip install -r requirements.txt
+        pip install --pre -U --extra-index-url https://test.pypi.org/simple/ PennyLane-Catalyst
+
+    - name: Run Python Pytest Tests with flaky Checks
+      run: |
+        make pytest ENABLE_FLAKY=ON

.github/workflows/notify-failed-jobs.yaml

@@ -10,6 +10,7 @@ on:
       - Build Catalyst Wheel on Linux (x86_64)
       - Build Catalyst Wheel on macOS (arm64)
       - Build nightly Catalyst releases for TestPyPI
+      - Run Catalyst pytests for weekly flaky test checking
 
 jobs:
   on-failure:

Makefile

@@ -21,6 +21,7 @@ TEST_BACKEND ?= "lightning.qubit"
 TEST_BRAKET ?= NONE
 ENABLE_ASAN ?= OFF
 TOML_SPECS ?= $(shell find ./runtime ./frontend -name '*.toml' -not -name 'pyproject.toml')
+ENABLE_FLAKY ?= OFF
 
 PLATFORM := $(shell uname -s)
 ifeq ($(PLATFORM),Linux)
@@ -54,7 +55,11 @@ endif
 # with the ASAN runtime. Since we don't exert much control over the "user" compiler, skip them.
 TEST_EXCLUDES := -k "not test_executable_generation"
 endif
-PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES)
+FLAKY :=
+ifeq ($(ENABLE_FLAKY),ON)
+FLAKY := --force-flaky --max-runs=5 --min-passes=5
+endif
+PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES) $(FLAKY)
 
 # TODO: Find out why we have container overflow on macOS.
 ASAN_OPTIONS := ASAN_OPTIONS="detect_leaks=0,detect_container_overflow=0"

frontend/test/pytest/device/test_decomposition.py

@@ -90,7 +90,6 @@ class NoUnitaryDevice(qml.devices.Device):
 
     def __init__(self, shots=None, wires=None):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("QubitUnitary")
         self.qjit_capabilities = self.capabilities
 
     def apply(self, operations, **kwargs):
@@ -113,6 +112,9 @@ def execute(self, circuits, execution_config):
         return circuits, execution_config
 
 
+NoUnitaryDevice.capabilities.operations.pop("QubitUnitary")
+
+
 class TestControlledDecomposition:
     """Test behaviour around the decomposition of the `Controlled` class."""
 

frontend/test/pytest/test_autograph.py

@@ -55,7 +55,34 @@
 
 
 class Failing:
-    """Test class that emulates failures in user-code"""
+    """
+    Test class that emulates failures in user-code.
+
+    When autograph fails to convert, it will fall back to native python control flow execution.
+    In such cases autograph would raise a warning.
+
+    We want to test that this warning is properly raised, but if we use an actual error to trigger
+    the autograph fallback, the fallback would also fail. Therefore, we raise an exception in a
+    controlled fashion.
+
+    This Failing class has a class-level dictionary to keep track of what labels it has already
+    seen. When it sees a new label for the first time, it raises an exception. In all other cases,
+    it just silently lets the input value flow through.
+
+    For example, consider this code
+        @qjit(autograph=True)
+        def f1():
+            acc = 0
+            while acc < 5:
+                acc = Failing(acc, "while").val + 1
+            return acc
+
+    When tracing, the first Failing instance will encounter an unseen label "while".
+    This raises an exception and fails autograph, causing it to fallback.
+    Then in the actual python while loop, future Failing instances will encounter the same "while"
+    label. Since the label is not new, no new exception is raised, and the test finishes without
+    errors.
+    """
 
     triggered = defaultdict(bool)
 
@@ -73,6 +100,13 @@ def val(self):
         return self.ref
 
 
+@pytest.fixture(autouse=True)
+def reset_Failing():
+    """Reset class variable on `Failing` class before each test"""
+    Failing.triggered = defaultdict(bool)
+    yield
+
+
 class TestSourceCodeInfo:
     """Unit tests for exception utilities that retrieves traceback information for the original
     source code."""
@@ -1426,21 +1460,6 @@ def f1():
 
         assert f1() == sum([1, 1, 2, 2])
 
-    @pytest.mark.xfail(reason="this won't run warning-free until we fix the resource warning issue")
-    @pytest.mark.filterwarnings("error")
-    def test_whileloop_no_warning(self, monkeypatch):
-        """Test the absence of warnings if fallbacks are ignored."""
-        monkeypatch.setattr("catalyst.autograph_ignore_fallbacks", True)
-
-        @qjit(autograph=True)
-        def f():
-            acc = 0
-            while Failing(acc).val < 5:
-                acc = acc + 1
-            return acc
-
-        assert f() == 5
-
     def test_whileloop_exception(self, monkeypatch):
         """Test for-loop error if strict-conversion is enabled."""
         monkeypatch.setattr("catalyst.autograph_strict_conversion", True)

frontend/test/pytest/test_measurement_transforms.py

@@ -50,7 +50,6 @@ class CustomDevice(Device):
 
     def __init__(self, wires, shots=1024):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("BlockEncode")
 
     @staticmethod
     def get_c_interface():
@@ -69,6 +68,9 @@ def execute(self, circuits, execution_config):
         return circuits, execution_config
 
 
+CustomDevice.capabilities.operations.pop("BlockEncode")
+
+
 class CustomDeviceLimitedMPs(Device):
     """A Custom Device from the device API without wires."""
 

frontend/test/pytest/test_preprocess.py

@@ -87,7 +87,6 @@ class CustomDevice(Device):
 
     def __init__(self, wires, shots=1024):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("BlockEncode")
         self.qjit_capabilities = self.capabilities
 
     @staticmethod
@@ -106,6 +105,9 @@ def execute(self, circuits, execution_config):
         raise NotImplementedError
 
 
+CustomDevice.capabilities.operations.pop("BlockEncode")
+
+
 class TestDecomposition:
     """Test the preprocessing transforms implemented in Catalyst."""