Add flaky pytest infrastructure and weekend runners

.github/workflows/check-weekly-flaky.yaml

@@ -0,0 +1,32 @@
+name: Run Catalyst pytests for flaky test checking
+
+on:
+  pull_request: # temporarily set this on PR to check this script
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+  schedule:
+    # Run every weekend on Saturday at 23:40 EDT (cron is in UTC)
+    # https://crontab.cronhub.io/
+    - cron: "40 23 * * SAT"
+  workflow_dispatch:
+
+jobs:
+  install-dependencies:
+    name: Install dependencies
+    runs-on: ubuntu-24.04
+
+    steps:
+    - name: Checkout Catalyst repo
+      uses: actions/checkout@v4
+
+    - name: Install Deps
+      run: |
+        python3 -m pip install -r requirements.txt
+        pip install --pre -U --extra-index-url https://test.pypi.org/simple/ PennyLane-Catalyst
+
+    - name: Run Python Pytest Tests with flaky Checks
+      run: |
+        make pytest ENABLE_FLAKY=ON

Makefile

@@ -21,6 +21,7 @@ TEST_BACKEND ?= "lightning.qubit"
 TEST_BRAKET ?= NONE
 ENABLE_ASAN ?= OFF
 TOML_SPECS ?= $(shell find ./runtime ./frontend -name '*.toml' -not -name 'pyproject.toml')
+ENABLE_FLAKY ?= OFF
 
 PLATFORM := $(shell uname -s)
 ifeq ($(PLATFORM),Linux)
@@ -54,7 +55,11 @@ endif
 # with the ASAN runtime. Since we don't exert much control over the "user" compiler, skip them.
 TEST_EXCLUDES := -k "not test_executable_generation"
 endif
-PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES)
+FLAKY :=
+ifeq ($(ENABLE_FLAKY),ON)
+FLAKY := --force-flaky --max-runs=5 --min-passes=5
+endif
+PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES) $(FLAKY)
 
 # TODO: Find out why we have container overflow on macOS.
 ASAN_OPTIONS := ASAN_OPTIONS="detect_leaks=0,detect_container_overflow=0"

frontend/test/pytest/device/test_decomposition.py

@@ -21,6 +21,7 @@
 import numpy as np
 import pennylane as qml
 import pytest
+from flaky import flaky
 from pennylane.devices.capabilities import DeviceCapabilities, OperatorProperties
 
 from catalyst import CompileError, ctrl, qjit
@@ -136,6 +137,7 @@ def f():
         with pytest.raises(CompileError, match="could not be decomposed, it might be unsupported."):
             qjit(f, target="jaxpr")
 
+    @flaky(max_runs=1, min_passes=1)
     def test_no_unitary_support(self):
         """Test that unknown controlled operations without QubitUnitary support raise an error."""
 

frontend/test/pytest/test_autograph.py

@@ -24,6 +24,7 @@
 import numpy as np
 import pennylane as qml
 import pytest
+from flaky import flaky
 from jax.errors import TracerBoolConversionError
 from numpy.testing import assert_allclose
 
@@ -1685,6 +1686,7 @@ def test_logical_mixture_static_dynamic_default(self, s, d):
 class TestMixed:
     """Test a mix of supported autograph conversions and Catalyst control flow."""
 
+    @flaky(max_runs=1, min_passes=1)
     def test_force_python_fallbacks(self):
         """Test fallback modes of control-flow primitives."""
 

frontend/test/pytest/test_measurement_transforms.py

@@ -25,6 +25,7 @@
 import pennylane as qml
 import pytest
 from conftest import CONFIG_CUSTOM_DEVICE
+from flaky import flaky
 from pennylane.devices import Device
 from pennylane.devices.capabilities import OperatorProperties
 from pennylane.transforms import split_non_commuting, split_to_single_terms
@@ -741,6 +742,7 @@ def test_split_non_commuting_is_added_for_full_diagonalization(
 
         assert split_non_commuting in transform_program
 
+    @flaky(max_runs=1, min_passes=1)
     def test_measurements_are_split(self, mocker):
         """Test that the split_to_single_terms or split_non_commuting transform
         are added to the transform program from preprocess as expected, based on the

frontend/test/pytest/test_preprocess.py

@@ -19,6 +19,7 @@
 import pennylane as qml
 import pytest
 from conftest import CONFIG_CUSTOM_DEVICE
+from flaky import flaky
 from pennylane.devices import Device, NullQubit
 from pennylane.devices.capabilities import DeviceCapabilities, OperatorProperties
 from pennylane.tape import QuantumScript
@@ -136,6 +137,7 @@ def test_decompose_ops_to_unitary(self):
         assert isinstance(decomposed_ops[0], qml.QubitUnitary)
         assert isinstance(decomposed_ops[1], qml.RX)
 
+    @flaky(max_runs=1, min_passes=1)
     def test_decompose_ops_to_unitary_integration(self):
         """Test the decompose ops to unitary transform as part of the Catalyst pipeline."""
         dev = CustomDevice(wires=4, shots=None)