Add flaky pytest infrastructure and weekend runners

.github/workflows/check-weekly-flaky.yaml

@@ -0,0 +1,35 @@
+name: Run Catalyst pytests for flaky test checking
+
+permissions:
+  contents: read
+
+on:
+  pull_request: # temporarily set this on PR to check this script
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+  schedule:
+    # Run every weekend on Saturday at 23:40 EDT (cron is in UTC)
+    # https://crontab.cronhub.io/
+    - cron: "40 23 * * SAT"
+  workflow_dispatch:
+
+jobs:
+  install-dependencies:
+    name: Install dependencies
+    runs-on: ubuntu-24.04
+
+    steps:
+    - name: Checkout Catalyst repo
+      uses: actions/checkout@v4
+
+    - name: Install Deps
+      run: |
+        python3 -m pip install -r requirements.txt
+        pip install --pre -U --extra-index-url https://test.pypi.org/simple/ PennyLane-Catalyst
+
+    - name: Run Python Pytest Tests with flaky Checks
+      run: |
+        make pytest ENABLE_FLAKY=ON

Makefile

@@ -21,6 +21,7 @@ TEST_BACKEND ?= "lightning.qubit"
 TEST_BRAKET ?= NONE
 ENABLE_ASAN ?= OFF
 TOML_SPECS ?= $(shell find ./runtime ./frontend -name '*.toml' -not -name 'pyproject.toml')
+ENABLE_FLAKY ?= OFF
 
 PLATFORM := $(shell uname -s)
 ifeq ($(PLATFORM),Linux)
@@ -54,7 +55,11 @@ endif
 # with the ASAN runtime. Since we don't exert much control over the "user" compiler, skip them.
 TEST_EXCLUDES := -k "not test_executable_generation"
 endif
-PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES)
+FLAKY :=
+ifeq ($(ENABLE_FLAKY),ON)
+FLAKY := --force-flaky --max-runs=5 --min-passes=5
+endif
+PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES) $(FLAKY)
 
 # TODO: Find out why we have container overflow on macOS.
 ASAN_OPTIONS := ASAN_OPTIONS="detect_leaks=0,detect_container_overflow=0"

frontend/test/pytest/device/test_decomposition.py

@@ -90,7 +90,6 @@ class NoUnitaryDevice(qml.devices.Device):
 
     def __init__(self, shots=None, wires=None):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("QubitUnitary")
         self.qjit_capabilities = self.capabilities
 
     def apply(self, operations, **kwargs):
@@ -113,6 +112,9 @@ def execute(self, circuits, execution_config):
         return circuits, execution_config
 
 
+NoUnitaryDevice.capabilities.operations.pop("QubitUnitary")
+
+
 class TestControlledDecomposition:
     """Test behaviour around the decomposition of the `Controlled` class."""
 

frontend/test/pytest/test_autograph.py

@@ -24,6 +24,7 @@
 import numpy as np
 import pennylane as qml
 import pytest
+from flaky import flaky
 from jax.errors import TracerBoolConversionError
 from numpy.testing import assert_allclose
 
@@ -1685,6 +1686,7 @@ def test_logical_mixture_static_dynamic_default(self, s, d):
 class TestMixed:
     """Test a mix of supported autograph conversions and Catalyst control flow."""
 
+    @flaky(max_runs=1, min_passes=1)
     def test_force_python_fallbacks(self):
         """Test fallback modes of control-flow primitives."""
 

frontend/test/pytest/test_measurement_transforms.py

@@ -50,7 +50,6 @@ class CustomDevice(Device):
 
     def __init__(self, wires, shots=1024):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("BlockEncode")
 
     @staticmethod
     def get_c_interface():
@@ -69,6 +68,9 @@ def execute(self, circuits, execution_config):
         return circuits, execution_config
 
 
+CustomDevice.capabilities.operations.pop("BlockEncode")
+
+
 class CustomDeviceLimitedMPs(Device):
     """A Custom Device from the device API without wires."""
 

frontend/test/pytest/test_preprocess.py

@@ -87,7 +87,6 @@ class CustomDevice(Device):
 
     def __init__(self, wires, shots=1024):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("BlockEncode")
         self.qjit_capabilities = self.capabilities
 
     @staticmethod
@@ -106,6 +105,9 @@ def execute(self, circuits, execution_config):
         raise NotImplementedError
 
 
+CustomDevice.capabilities.operations.pop("BlockEncode")
+
+
 class TestDecomposition:
     """Test the preprocessing transforms implemented in Catalyst."""