From bb4634e70de4f76ec79e32b70eaabe393a6008f1 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Tue, 10 Jun 2025 15:52:44 -0400
Subject: [PATCH 01/21] Add flaky test infra in `make pytest`

---
 Makefile                                            | 7 ++++++-
 frontend/test/pytest/device/test_decomposition.py   | 2 ++
 frontend/test/pytest/test_autograph.py              | 2 ++
 frontend/test/pytest/test_measurement_transforms.py | 2 ++
 frontend/test/pytest/test_preprocess.py             | 2 ++
 5 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1552c59f1c..ff3cb1f0f7 100644
--- a/Makefile
+++ b/Makefile
@@ -21,6 +21,7 @@ TEST_BACKEND ?= "lightning.qubit"
 TEST_BRAKET ?= NONE
 ENABLE_ASAN ?= OFF
 TOML_SPECS ?= $(shell find ./runtime ./frontend -name '*.toml' -not -name 'pyproject.toml')
+ENABLE_FLAKY ?= OFF
 
 PLATFORM := $(shell uname -s)
 ifeq ($(PLATFORM),Linux)
@@ -54,7 +55,11 @@ endif
 # with the ASAN runtime. Since we don't exert much control over the "user" compiler, skip them.
 TEST_EXCLUDES := -k "not test_executable_generation"
 endif
-PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES)
+FLAKY :=
+ifeq ($(ENABLE_FLAKY),ON)
+FLAKY := --force-flaky --max-runs=2 --min-passes=2
+endif
+PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES) $(FLAKY)
 
 # TODO: Find out why we have container overflow on macOS.
 ASAN_OPTIONS := ASAN_OPTIONS="detect_leaks=0,detect_container_overflow=0"
diff --git a/frontend/test/pytest/device/test_decomposition.py b/frontend/test/pytest/device/test_decomposition.py
index adbd571cf4..9011dfad79 100644
--- a/frontend/test/pytest/device/test_decomposition.py
+++ b/frontend/test/pytest/device/test_decomposition.py
@@ -21,6 +21,7 @@
 import numpy as np
 import pennylane as qml
 import pytest
+from flaky import flaky
 from pennylane.devices.capabilities import DeviceCapabilities, OperatorProperties
 
 from catalyst import CompileError, ctrl, qjit
@@ -136,6 +137,7 @@ def f():
         with pytest.raises(CompileError, match="could not be decomposed, it might be unsupported."):
             qjit(f, target="jaxpr")
 
+    @flaky(max_runs=1, min_passes=1)
     def test_no_unitary_support(self):
         """Test that unknown controlled operations without QubitUnitary support raise an error."""
 
diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index 21dbc65c21..845ccc9d43 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -24,6 +24,7 @@
 import numpy as np
 import pennylane as qml
 import pytest
+from flaky import flaky
 from jax.errors import TracerBoolConversionError
 from numpy.testing import assert_allclose
 
@@ -1685,6 +1686,7 @@ def test_logical_mixture_static_dynamic_default(self, s, d):
 class TestMixed:
     """Test a mix of supported autograph conversions and Catalyst control flow."""
 
+    @flaky(max_runs=1, min_passes=1)
     def test_force_python_fallbacks(self):
         """Test fallback modes of control-flow primitives."""
 
diff --git a/frontend/test/pytest/test_measurement_transforms.py b/frontend/test/pytest/test_measurement_transforms.py
index 17807ecc55..d173587f8e 100644
--- a/frontend/test/pytest/test_measurement_transforms.py
+++ b/frontend/test/pytest/test_measurement_transforms.py
@@ -25,6 +25,7 @@
 import pennylane as qml
 import pytest
 from conftest import CONFIG_CUSTOM_DEVICE
+from flaky import flaky
 from pennylane.devices import Device
 from pennylane.devices.capabilities import OperatorProperties
 from pennylane.transforms import split_non_commuting, split_to_single_terms
@@ -741,6 +742,7 @@ def test_split_non_commuting_is_added_for_full_diagonalization(
 
         assert split_non_commuting in transform_program
 
+    @flaky(max_runs=1, min_passes=1)
     def test_measurements_are_split(self, mocker):
         """Test that the split_to_single_terms or split_non_commuting transform
         are added to the transform program from preprocess as expected, based on the
diff --git a/frontend/test/pytest/test_preprocess.py b/frontend/test/pytest/test_preprocess.py
index 2f8ab8209c..f77cabc9d0 100644
--- a/frontend/test/pytest/test_preprocess.py
+++ b/frontend/test/pytest/test_preprocess.py
@@ -19,6 +19,7 @@
 import pennylane as qml
 import pytest
 from conftest import CONFIG_CUSTOM_DEVICE
+from flaky import flaky
 from pennylane.devices import Device, NullQubit
 from pennylane.devices.capabilities import DeviceCapabilities, OperatorProperties
 from pennylane.tape import QuantumScript
@@ -136,6 +137,7 @@ def test_decompose_ops_to_unitary(self):
         assert isinstance(decomposed_ops[0], qml.QubitUnitary)
         assert isinstance(decomposed_ops[1], qml.RX)
 
+    @flaky(max_runs=1, min_passes=1)
     def test_decompose_ops_to_unitary_integration(self):
         """Test the decompose ops to unitary transform as part of the Catalyst pipeline."""
         dev = CustomDevice(wires=4, shots=None)

From 931db6a0057ca5314b1752b9b92ef6d3bc2092de Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Tue, 10 Jun 2025 15:53:09 -0400
Subject: [PATCH 02/21] 5 runs

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ff3cb1f0f7..a5b42d0968 100644
--- a/Makefile
+++ b/Makefile
@@ -57,7 +57,7 @@ TEST_EXCLUDES := -k "not test_executable_generation"
 endif
 FLAKY :=
 ifeq ($(ENABLE_FLAKY),ON)
-FLAKY := --force-flaky --max-runs=2 --min-passes=2
+FLAKY := --force-flaky --max-runs=5 --min-passes=5
 endif
 PYTEST_FLAGS := $(PARALLELIZE) $(TEST_EXCLUDES) $(FLAKY)
 

From d61ea62d347754b66706e8b2539ee3f7c9d37f03 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Tue, 10 Jun 2025 16:23:29 -0400
Subject: [PATCH 03/21] add weekly flaky test runner script

---
 .github/workflows/check-weekly-flaky.yaml | 36 +++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 .github/workflows/check-weekly-flaky.yaml

diff --git a/.github/workflows/check-weekly-flaky.yaml b/.github/workflows/check-weekly-flaky.yaml
new file mode 100644
index 0000000000..73974c072d
--- /dev/null
+++ b/.github/workflows/check-weekly-flaky.yaml
@@ -0,0 +1,36 @@
+name: Run Catalyst pytests for flaky test checking
+
+on:
+  pull_request: # temporarily set this on PR to check this script
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+  schedule:
+    # Run every weekend on Saturday at 23:40 EDT (cron is in UTC)
+    # https://crontab.cronhub.io/
+    - cron: "40 23 * * SAT"
+  workflow_dispatch:
+
+jobs:
+  install-dependencies:
+    name: Install dependencies
+    runs-on: ubuntu-24.04
+
+    steps:
+    - name: Checkout Catalyst repo
+      uses: actions/checkout@v4
+
+    - name: Install Deps
+      run: |
+        python3 -m pip install -r requirements.txt
+        python3 -m pip install cuda-quantum==0.6.0
+        python3 -m pip install oqc-qcaas-client
+        make frontend
+        pip uninstall pennylane-catalyst
+        pip install -i https://test.pypi.org/simple/ PennyLane-Catalyst
+
+    - name: Run Python Pytest Tests with flaky Checks
+      run: |
+        make pytest ENABLE_FLAKY=ON

From cc61117c04166b274a094e4cbc2a09c915fc8dcf Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Tue, 10 Jun 2025 17:10:01 -0400
Subject: [PATCH 04/21] try skipping cuda

---
 .github/workflows/check-weekly-flaky.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/check-weekly-flaky.yaml b/.github/workflows/check-weekly-flaky.yaml
index 73974c072d..48cf088831 100644
--- a/.github/workflows/check-weekly-flaky.yaml
+++ b/.github/workflows/check-weekly-flaky.yaml
@@ -25,7 +25,7 @@ jobs:
     - name: Install Deps
       run: |
         python3 -m pip install -r requirements.txt
-        python3 -m pip install cuda-quantum==0.6.0
+        #python3 -m pip install cuda-quantum==0.6.0
         python3 -m pip install oqc-qcaas-client
         make frontend
         pip uninstall pennylane-catalyst

From 298a9ab30bf96f8540abac256e9648527afe01db Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Tue, 10 Jun 2025 17:17:56 -0400
Subject: [PATCH 05/21] don't ask when pip uninstall

---
 .github/workflows/check-weekly-flaky.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/check-weekly-flaky.yaml b/.github/workflows/check-weekly-flaky.yaml
index 48cf088831..a17f11103d 100644
--- a/.github/workflows/check-weekly-flaky.yaml
+++ b/.github/workflows/check-weekly-flaky.yaml
@@ -28,7 +28,7 @@ jobs:
         #python3 -m pip install cuda-quantum==0.6.0
         python3 -m pip install oqc-qcaas-client
         make frontend
-        pip uninstall pennylane-catalyst
+        pip uninstall -y pennylane-catalyst
         pip install -i https://test.pypi.org/simple/ PennyLane-Catalyst
 
     - name: Run Python Pytest Tests with flaky Checks

From 41ca911e21a0b250a54b33d11ff2141ea0747efc Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Tue, 10 Jun 2025 17:36:16 -0400
Subject: [PATCH 06/21] testpypi catalyst follows dep versions already

---
 .github/workflows/check-weekly-flaky.yaml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/check-weekly-flaky.yaml b/.github/workflows/check-weekly-flaky.yaml
index a17f11103d..1782a432ff 100644
--- a/.github/workflows/check-weekly-flaky.yaml
+++ b/.github/workflows/check-weekly-flaky.yaml
@@ -25,11 +25,7 @@ jobs:
     - name: Install Deps
       run: |
         python3 -m pip install -r requirements.txt
-        #python3 -m pip install cuda-quantum==0.6.0
-        python3 -m pip install oqc-qcaas-client
-        make frontend
-        pip uninstall -y pennylane-catalyst
-        pip install -i https://test.pypi.org/simple/ PennyLane-Catalyst
+        pip install --pre -U --extra-index-url https://test.pypi.org/simple/ PennyLane-Catalyst
 
     - name: Run Python Pytest Tests with flaky Checks
       run: |

From fdcd8f3f1d855ab49d28a671c9519d723eba77e9 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Wed, 11 Jun 2025 13:23:08 -0400
Subject: [PATCH 07/21] set read-only permission on job (suggestion by github
 action bot)

---
 .github/workflows/check-weekly-flaky.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/check-weekly-flaky.yaml b/.github/workflows/check-weekly-flaky.yaml
index 1782a432ff..2c0c9da033 100644
--- a/.github/workflows/check-weekly-flaky.yaml
+++ b/.github/workflows/check-weekly-flaky.yaml
@@ -1,5 +1,8 @@
 name: Run Catalyst pytests for flaky test checking
 
+permissions:
+  contents: read
+
 on:
   pull_request: # temporarily set this on PR to check this script
     types:

From b61f2db949b2f9046da809489b67c3f352e5a908 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Wed, 11 Jun 2025 15:24:52 -0400
Subject: [PATCH 08/21] popping capabilities should be per custom device class,
 NOT per instance!

---
 frontend/test/pytest/device/test_decomposition.py   | 5 +++--
 frontend/test/pytest/test_measurement_transforms.py | 6 +++---
 frontend/test/pytest/test_preprocess.py             | 5 +++--
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/frontend/test/pytest/device/test_decomposition.py b/frontend/test/pytest/device/test_decomposition.py
index 9011dfad79..cdc93d30d3 100644
--- a/frontend/test/pytest/device/test_decomposition.py
+++ b/frontend/test/pytest/device/test_decomposition.py
@@ -91,7 +91,6 @@ class NoUnitaryDevice(qml.devices.Device):
 
     def __init__(self, shots=None, wires=None):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("QubitUnitary")
         self.qjit_capabilities = self.capabilities
 
     def apply(self, operations, **kwargs):
@@ -114,6 +113,9 @@ def execute(self, circuits, execution_config):
         return circuits, execution_config
 
 
+NoUnitaryDevice.capabilities.operations.pop("QubitUnitary")
+
+
 class TestControlledDecomposition:
     """Test behaviour around the decomposition of the `Controlled` class."""
 
@@ -137,7 +139,6 @@ def f():
         with pytest.raises(CompileError, match="could not be decomposed, it might be unsupported."):
             qjit(f, target="jaxpr")
 
-    @flaky(max_runs=1, min_passes=1)
     def test_no_unitary_support(self):
         """Test that unknown controlled operations without QubitUnitary support raise an error."""
 
diff --git a/frontend/test/pytest/test_measurement_transforms.py b/frontend/test/pytest/test_measurement_transforms.py
index d173587f8e..edc663da88 100644
--- a/frontend/test/pytest/test_measurement_transforms.py
+++ b/frontend/test/pytest/test_measurement_transforms.py
@@ -25,7 +25,6 @@
 import pennylane as qml
 import pytest
 from conftest import CONFIG_CUSTOM_DEVICE
-from flaky import flaky
 from pennylane.devices import Device
 from pennylane.devices.capabilities import OperatorProperties
 from pennylane.transforms import split_non_commuting, split_to_single_terms
@@ -51,7 +50,6 @@ class CustomDevice(Device):
 
     def __init__(self, wires, shots=1024):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("BlockEncode")
 
     @staticmethod
     def get_c_interface():
@@ -70,6 +68,9 @@ def execute(self, circuits, execution_config):
         return circuits, execution_config
 
 
+CustomDevice.capabilities.operations.pop("BlockEncode")
+
+
 class CustomDeviceLimitedMPs(Device):
     """A Custom Device from the device API without wires."""
 
@@ -742,7 +743,6 @@ def test_split_non_commuting_is_added_for_full_diagonalization(
 
         assert split_non_commuting in transform_program
 
-    @flaky(max_runs=1, min_passes=1)
     def test_measurements_are_split(self, mocker):
         """Test that the split_to_single_terms or split_non_commuting transform
         are added to the transform program from preprocess as expected, based on the
diff --git a/frontend/test/pytest/test_preprocess.py b/frontend/test/pytest/test_preprocess.py
index f77cabc9d0..9bf9d57f08 100644
--- a/frontend/test/pytest/test_preprocess.py
+++ b/frontend/test/pytest/test_preprocess.py
@@ -88,7 +88,6 @@ class CustomDevice(Device):
 
     def __init__(self, wires, shots=1024):
         super().__init__(wires=wires, shots=shots)
-        self.capabilities.operations.pop("BlockEncode")
         self.qjit_capabilities = self.capabilities
 
     @staticmethod
@@ -107,6 +106,9 @@ def execute(self, circuits, execution_config):
         raise NotImplementedError
 
 
+CustomDevice.capabilities.operations.pop("BlockEncode")
+
+
 class TestDecomposition:
     """Test the preprocessing transforms implemented in Catalyst."""
 
@@ -137,7 +139,6 @@ def test_decompose_ops_to_unitary(self):
         assert isinstance(decomposed_ops[0], qml.QubitUnitary)
         assert isinstance(decomposed_ops[1], qml.RX)
 
-    @flaky(max_runs=1, min_passes=1)
     def test_decompose_ops_to_unitary_integration(self):
         """Test the decompose ops to unitary transform as part of the Catalyst pipeline."""
         dev = CustomDevice(wires=4, shots=None)

From a981a4a9824a2a415bee63e4550a5dee10b53ec9 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Wed, 11 Jun 2025 15:34:21 -0400
Subject: [PATCH 09/21] remove unused flaky import

---
 frontend/test/pytest/device/test_decomposition.py | 1 -
 frontend/test/pytest/test_preprocess.py           | 1 -
 2 files changed, 2 deletions(-)

diff --git a/frontend/test/pytest/device/test_decomposition.py b/frontend/test/pytest/device/test_decomposition.py
index cdc93d30d3..00356a7440 100644
--- a/frontend/test/pytest/device/test_decomposition.py
+++ b/frontend/test/pytest/device/test_decomposition.py
@@ -21,7 +21,6 @@
 import numpy as np
 import pennylane as qml
 import pytest
-from flaky import flaky
 from pennylane.devices.capabilities import DeviceCapabilities, OperatorProperties
 
 from catalyst import CompileError, ctrl, qjit
diff --git a/frontend/test/pytest/test_preprocess.py b/frontend/test/pytest/test_preprocess.py
index 9bf9d57f08..ca684b2e62 100644
--- a/frontend/test/pytest/test_preprocess.py
+++ b/frontend/test/pytest/test_preprocess.py
@@ -19,7 +19,6 @@
 import pennylane as qml
 import pytest
 from conftest import CONFIG_CUSTOM_DEVICE
-from flaky import flaky
 from pennylane.devices import Device, NullQubit
 from pennylane.devices.capabilities import DeviceCapabilities, OperatorProperties
 from pennylane.tape import QuantumScript

From 38b875362220f901c70f22261fe9e41ebae58e69 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Wed, 11 Jun 2025 15:52:02 -0400
Subject: [PATCH 10/21] add reset fixture to autograph `Failing` test util
 class

---
 frontend/test/pytest/test_autograph.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index 845ccc9d43..9f56d16c51 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -24,7 +24,6 @@
 import numpy as np
 import pennylane as qml
 import pytest
-from flaky import flaky
 from jax.errors import TracerBoolConversionError
 from numpy.testing import assert_allclose
 
@@ -74,6 +73,13 @@ def val(self):
         return self.ref
 
 
+@pytest.fixture(autouse=True)
+def reset_Failing():
+    save = Failing.triggered.copy()
+    yield
+    Failing.triggered = save
+
+
 class TestSourceCodeInfo:
     """Unit tests for exception utilities that retrieves traceback information for the original
     source code."""
@@ -1686,7 +1692,6 @@ def test_logical_mixture_static_dynamic_default(self, s, d):
 class TestMixed:
     """Test a mix of supported autograph conversions and Catalyst control flow."""
 
-    @flaky(max_runs=1, min_passes=1)
     def test_force_python_fallbacks(self):
         """Test fallback modes of control-flow primitives."""
 

From 35feb0f3afc0c3dbf2589c69c3e32ccc9bf6d719 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Wed, 11 Jun 2025 15:57:32 -0400
Subject: [PATCH 11/21] add failure notif

---
 .github/workflows/check-weekly-flaky.yaml |  6 +++---
 .github/workflows/notify-failed-jobs.yaml | 10 ++++++----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/check-weekly-flaky.yaml b/.github/workflows/check-weekly-flaky.yaml
index 2c0c9da033..5888a54fdc 100644
--- a/.github/workflows/check-weekly-flaky.yaml
+++ b/.github/workflows/check-weekly-flaky.yaml
@@ -1,4 +1,4 @@
-name: Run Catalyst pytests for flaky test checking
+name: Run Catalyst pytests for weekly flaky test checking
 
 permissions:
   contents: read
@@ -17,8 +17,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  install-dependencies:
-    name: Install dependencies
+  weekly-flaky-test-run:
+    name: Run weekly flaky tests
     runs-on: ubuntu-24.04
 
     steps:
diff --git a/.github/workflows/notify-failed-jobs.yaml b/.github/workflows/notify-failed-jobs.yaml
index 39998cc9bd..ce7ddb1490 100644
--- a/.github/workflows/notify-failed-jobs.yaml
+++ b/.github/workflows/notify-failed-jobs.yaml
@@ -3,13 +3,15 @@ on:
   workflow_run:
     branches:
       - main
+      - paul0403/add_flaky_runs # just testing, remove later
     types:
       - completed
     workflows:
-      - Build Catalyst Wheel on Linux (arm64)
-      - Build Catalyst Wheel on Linux (x86_64)
-      - Build Catalyst Wheel on macOS (arm64)
-      - Build nightly Catalyst releases for TestPyPI
+      # - Build Catalyst Wheel on Linux (arm64)
+      # - Build Catalyst Wheel on Linux (x86_64)
+      # - Build Catalyst Wheel on macOS (arm64)
+      # - Build nightly Catalyst releases for TestPyPI
+      - Run Catalyst pytests for weekly flaky test checking
 
 jobs:
   on-failure:

From 4d78c8e987124dd808282e4abaceaf22732a036d Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Wed, 11 Jun 2025 16:15:32 -0400
Subject: [PATCH 12/21] codefactor

---
 frontend/test/pytest/test_autograph.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index 9f56d16c51..e3736dbb58 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -75,6 +75,7 @@ def val(self):
 
 @pytest.fixture(autouse=True)
 def reset_Failing():
+    """Reset class variable on `Failing` class after each test"""
     save = Failing.triggered.copy()
     yield
     Failing.triggered = save

From a804c32811dc95f570fe7dc4660f7c2b02a8e980 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Wed, 11 Jun 2025 16:25:34 -0400
Subject: [PATCH 13/21] test failure notif

---
 .github/workflows/notify-failed-jobs.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/notify-failed-jobs.yaml b/.github/workflows/notify-failed-jobs.yaml
index ce7ddb1490..bfc8dd505d 100644
--- a/.github/workflows/notify-failed-jobs.yaml
+++ b/.github/workflows/notify-failed-jobs.yaml
@@ -3,7 +3,7 @@ on:
   workflow_run:
     branches:
       - main
-      - paul0403/add_flaky_runs # just testing, remove later
+      - refs/remotes/pull/1799/merge # just testing, remove later
     types:
       - completed
     workflows:

From 29e82b8e540b5e260902a22498e8b5a12dd17114 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Wed, 11 Jun 2025 16:51:20 -0400
Subject: [PATCH 14/21] remove temporary PR testings

---
 .github/workflows/check-weekly-flaky.yaml | 6 ------
 .github/workflows/notify-failed-jobs.yaml | 9 ++++-----
 2 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/check-weekly-flaky.yaml b/.github/workflows/check-weekly-flaky.yaml
index 5888a54fdc..a798d6519d 100644
--- a/.github/workflows/check-weekly-flaky.yaml
+++ b/.github/workflows/check-weekly-flaky.yaml
@@ -4,12 +4,6 @@ permissions:
   contents: read
 
 on:
-  pull_request: # temporarily set this on PR to check this script
-    types:
-      - opened
-      - reopened
-      - synchronize
-      - ready_for_review
   schedule:
     # Run every weekend on Saturday at 23:40 EDT (cron is in UTC)
     # https://crontab.cronhub.io/
diff --git a/.github/workflows/notify-failed-jobs.yaml b/.github/workflows/notify-failed-jobs.yaml
index bfc8dd505d..74f4305a12 100644
--- a/.github/workflows/notify-failed-jobs.yaml
+++ b/.github/workflows/notify-failed-jobs.yaml
@@ -3,14 +3,13 @@ on:
   workflow_run:
     branches:
       - main
-      - refs/remotes/pull/1799/merge # just testing, remove later
     types:
       - completed
     workflows:
-      # - Build Catalyst Wheel on Linux (arm64)
-      # - Build Catalyst Wheel on Linux (x86_64)
-      # - Build Catalyst Wheel on macOS (arm64)
-      # - Build nightly Catalyst releases for TestPyPI
+      - Build Catalyst Wheel on Linux (arm64)
+      - Build Catalyst Wheel on Linux (x86_64)
+      - Build Catalyst Wheel on macOS (arm64)
+      - Build nightly Catalyst releases for TestPyPI
       - Run Catalyst pytests for weekly flaky test checking
 
 jobs:

From 86755fb4005ebc3ff257aa58ee9d48003e70dadd Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Thu, 12 Jun 2025 09:43:56 -0400
Subject: [PATCH 15/21] remove unnecessary line

---
 .github/workflows/check-weekly-flaky.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/check-weekly-flaky.yaml b/.github/workflows/check-weekly-flaky.yaml
index a798d6519d..a63dbbcf4d 100644
--- a/.github/workflows/check-weekly-flaky.yaml
+++ b/.github/workflows/check-weekly-flaky.yaml
@@ -8,7 +8,6 @@ on:
     # Run every weekend on Saturday at 23:40 EDT (cron is in UTC)
     # https://crontab.cronhub.io/
     - cron: "40 23 * * SAT"
-  workflow_dispatch:
 
 jobs:
   weekly-flaky-test-run:

From 05421efd35d6a00456d4d03ea88f585d5b7dfc77 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Thu, 12 Jun 2025 15:40:01 -0400
Subject: [PATCH 16/21] add docstring for the mysterious `Failing` class in
 autograph test

---
 frontend/test/pytest/test_autograph.py | 33 ++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index e3736dbb58..feff19d6a1 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -55,7 +55,33 @@
 
 
 class Failing:
-    """Test class that emulates failures in user-code"""
+    """
+    Test class that emulates failures in user-code.
+
+    When autograph fails to convert, it will fall back to native python control flow execution.
+    In such cases autograph would raise a warning.
+
+    We want to test that this warning is properly raised, but if we use an actual error to trigger
+    the autograph fallback, the pytest would fail. Therefore, we manually raise an exception.
+
+    This Failing class has a class-level dictionary to keep track of what labels it has already
+    seen. When it sees a new label for the first time, it raises an exception. In all other cases,
+    it just silently lets the input value flow through.
+
+    For example, consider this code
+        @qjit(autograph=True)
+        def f1():
+            acc = 0
+            while acc < 5:
+                acc = Failing(acc, "while").val + 1
+            return acc
+
+    When tracing, the first Failing instance will encounter an unseen label "while".
+    This raises an exception and fails autograph, causing it to fallback.
+    Then in the actual python while loop, future Failing instances will encounter the same "while"
+    label. Since the label is not new, no new exception is raised, and the test finishes without
+    errors.
+    """
 
     triggered = defaultdict(bool)
 
@@ -75,10 +101,9 @@ def val(self):
 
 @pytest.fixture(autouse=True)
 def reset_Failing():
-    """Reset class variable on `Failing` class after each test"""
-    save = Failing.triggered.copy()
+    """Reset class variable on `Failing` class before each test"""
+    Failing.triggered = defaultdict(bool)
     yield
-    Failing.triggered = save
 
 
 class TestSourceCodeInfo:

From 6b6a564649635d84403bbddd8ce286a1f83d597c Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Fri, 13 Jun 2025 11:01:37 -0400
Subject: [PATCH 17/21] test_whileloop_no_warning passes

---
 frontend/test/pytest/test_autograph.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index feff19d6a1..7d41147495 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -1459,11 +1459,9 @@ def f1():
 
         assert f1() == sum([1, 1, 2, 2])
 
-    @pytest.mark.xfail(reason="this won't run warning-free until we fix the resource warning issue")
     @pytest.mark.filterwarnings("error")
     def test_whileloop_no_warning(self, monkeypatch):
         """Test the absence of warnings if fallbacks are ignored."""
-        monkeypatch.setattr("catalyst.autograph_ignore_fallbacks", True)
 
         @qjit(autograph=True)
         def f():

From de5e9ab72af806c953de96773c6a1922be41ddc1 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Fri, 13 Jun 2025 11:02:38 -0400
Subject: [PATCH 18/21] remove unused argument

---
 frontend/test/pytest/test_autograph.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index 7d41147495..97b983fb59 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -1460,7 +1460,7 @@ def f1():
         assert f1() == sum([1, 1, 2, 2])
 
     @pytest.mark.filterwarnings("error")
-    def test_whileloop_no_warning(self, monkeypatch):
+    def test_whileloop_no_warning(self):
         """Test the absence of warnings if fallbacks are ignored."""
 
         @qjit(autograph=True)

From e4f5a799a1cec1a5ad531e4f2916973bac345c10 Mon Sep 17 00:00:00 2001
From: Paul <79805239+paul0403@users.noreply.github.com>
Date: Fri, 13 Jun 2025 13:19:52 -0400
Subject: [PATCH 19/21] Update frontend/test/pytest/test_autograph.py

Co-authored-by: David Ittah <dime10@users.noreply.github.com>
---
 frontend/test/pytest/test_autograph.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index 97b983fb59..aa01f4276d 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -62,7 +62,7 @@ class Failing:
     In such cases autograph would raise a warning.
 
     We want to test that this warning is properly raised, but if we use an actual error to trigger
-    the autograph fallback, the pytest would fail. Therefore, we manually raise an exception.
+    the autograph fallback, the fallback would also fail. Therefore, we raise an exception in a controlled fashion.
 
     This Failing class has a class-level dictionary to keep track of what labels it has already
     seen. When it sees a new label for the first time, it raises an exception. In all other cases,

From c8f795657d6ea4f00db37c4e5f7178012bd28b3d Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Fri, 13 Jun 2025 13:21:34 -0400
Subject: [PATCH 20/21] line too long

---
 frontend/test/pytest/test_autograph.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index aa01f4276d..27f37040ce 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -62,7 +62,8 @@ class Failing:
     In such cases autograph would raise a warning.
 
     We want to test that this warning is properly raised, but if we use an actual error to trigger
-    the autograph fallback, the fallback would also fail. Therefore, we raise an exception in a controlled fashion.
+    the autograph fallback, the fallback would also fail. Therefore, we raise an exception in a
+    controlled fashion.
 
     This Failing class has a class-level dictionary to keep track of what labels it has already
     seen. When it sees a new label for the first time, it raises an exception. In all other cases,

From d26d0435b554aef9dd5084d1c946f96f49b47f95 Mon Sep 17 00:00:00 2001
From: paul0403 <paulhaochen.wang@gmail.com>
Date: Fri, 13 Jun 2025 14:39:04 -0400
Subject: [PATCH 21/21] remove test_whileloop_no_warning

---
 frontend/test/pytest/test_autograph.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/frontend/test/pytest/test_autograph.py b/frontend/test/pytest/test_autograph.py
index 27f37040ce..3ff412b694 100644
--- a/frontend/test/pytest/test_autograph.py
+++ b/frontend/test/pytest/test_autograph.py
@@ -1460,19 +1460,6 @@ def f1():
 
         assert f1() == sum([1, 1, 2, 2])
 
-    @pytest.mark.filterwarnings("error")
-    def test_whileloop_no_warning(self):
-        """Test the absence of warnings if fallbacks are ignored."""
-
-        @qjit(autograph=True)
-        def f():
-            acc = 0
-            while Failing(acc).val < 5:
-                acc = acc + 1
-            return acc
-
-        assert f() == 5
-
     def test_whileloop_exception(self, monkeypatch):
         """Test for-loop error if strict-conversion is enabled."""
         monkeypatch.setattr("catalyst.autograph_strict_conversion", True)