[Helix] Only run flaky tests on quarantine runs (#18937)

* Only run flaky tests on quaratine run

* Only run flaky tests on quarantine run

* Rename to QUARANTINE

* Rename to QUARANTINE

Author: HaoK

eng/helix/content/runtests.cmd

@@ -40,9 +40,10 @@ if not errorlevel 1 (
 set exit_code=0
 
 set NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
+set QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
 if (%quarantined%==true) (
     echo Running all tests.
-    %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit
+    %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit --TestCaseFilter:%QUARANTINE_FILTER%
     if errorlevel 1 (
         echo Failure in flaky test 1>&2
         REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1

eng/helix/content/runtests.sh

@@ -104,9 +104,10 @@ exit_code=0
 # only puts the explicit filter traits the user provided in the flaky attribute
 # Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
 NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true"
+QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$helix_queue_name=true"
 if [ "$quarantined" == true ]; then
     echo "Running all tests including quarantined."
-    $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit
+    $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$QUARANTINE_FILTER"
     if [ $? != 0 ]; then
         echo "Quarantined tests failed!" 1>&2
         # DO NOT EXIT