#[Retry] attribute to support retrying flaky test

src/Framework/Attributes/Retry.php

@@ -0,0 +1,57 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\Framework\Attributes;
+
+use Attribute;
+
+/**
+ * @immutable
+ *
+ * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit
+ */
+#[Attribute(Attribute::TARGET_METHOD | Attribute::IS_REPEATABLE)]
+final readonly class Retry
+{
+    private int $maxRetries;
+    private int $delay;
+
+    /**
+     * @var ?non-empty-string
+     */
+    private ?string $retryOn;
+
+    /**
+     * @param ?non-empty-string $retryOn
+     */
+    public function __construct(int $maxRetries, ?int $delay = 0, ?string $retryOn = null)
+    {
+        $this->maxRetries = $maxRetries;
+        $this->delay      = $delay;
+        $this->retryOn    = $retryOn;
+    }
+
+    public function maxRetries(): int
+    {
+        return $this->maxRetries;
+    }
+
+    public function delay(): int
+    {
+        return $this->delay;
+    }
+
+    /**
+     * @return ?non-empty-string
+     */
+    public function retryOn(): ?string
+    {
+        return $this->retryOn;
+    }
+}

src/Framework/TestCase.php

@@ -46,6 +46,7 @@
 use function restore_exception_handler;
 use function set_error_handler;
 use function set_exception_handler;
+use function sleep;
 use function sprintf;
 use function str_contains;
 use function stream_get_contents;
@@ -79,6 +80,7 @@
 use PHPUnit\Metadata\Api\HookMethods;
 use PHPUnit\Metadata\Api\Requirements;
 use PHPUnit\Metadata\Parser\Registry as MetadataRegistry;
+use PHPUnit\Metadata\Retry;
 use PHPUnit\Metadata\WithEnvironmentVariable;
 use PHPUnit\Runner\BackedUpEnvironmentVariable;
 use PHPUnit\Runner\DeprecationCollector\Facade as DeprecationCollector;
@@ -1248,7 +1250,7 @@ protected function onNotSuccessfulTest(Throwable $t): never
      * @throws ExpectationFailedException
      * @throws Throwable
      */
-    private function runTest(): mixed
+    private function runTest(int $attempt = 0): mixed
     {
         $testArguments = array_merge($this->data, array_values($this->dependencyInput));
 
@@ -1276,6 +1278,12 @@ private function runTest(): mixed
             }
 
             if (!$this->shouldExceptionExpectationsBeVerified($exception)) {
+                $metadata = $this->getRetryMetadata($exception, $attempt);
+
+                if (null !== $metadata) {
+                    return $this->retryTest($metadata, $attempt);
+                }
+
                 throw $exception;
             }
 
@@ -2255,6 +2263,28 @@ private function handleExceptionFromInvokedCountMockObjectRule(Throwable $t): vo
         }
     }
 
+    private function getRetryMetadata(Throwable $th, int $attempt): ?Retry
+    {
+        foreach (MetadataRegistry::parser()->forMethod($this::class, $this->name())->isRetry() as $metadata) {
+            assert($metadata instanceof Retry);
+
+            if ($metadata->maxRetries() > $attempt && (null === $metadata->retryOn() || $th instanceof ($metadata->retryOn()))) {
+                return $metadata;
+            }
+        }
+
+        return null;
+    }
+
+    private function retryTest(Retry $metadata, int $attempt): mixed
+    {
+        if ($metadata->delay() > 0) {
+            sleep($metadata->delay());
+        }
+
+        return $this->runTest(++$attempt);
+    }
+
     /**
      * Creates a test stub for the specified interface or class.
      *

src/Metadata/Metadata.php

@@ -534,6 +534,11 @@ public static function withoutErrorHandler(): WithoutErrorHandler
         return new WithoutErrorHandler(self::METHOD_LEVEL);
     }
 
+    public static function retry(int $maxRetries, ?int $delay = 0, ?string $retryOn = null): Retry
+    {
+        return new Retry(self::METHOD_LEVEL, $maxRetries, $delay, $retryOn);
+    }
+
     /**
      * @param int<0, 1> $level
      */
@@ -969,4 +974,12 @@ public function isWithoutErrorHandler(): bool
     {
         return false;
     }
+
+    /**
+     * @phpstan-assert-if-true Retry $this
+     */
+    public function isRetry(): bool
+    {
+        return false;
+    }
 }

src/Metadata/MetadataCollection.php

@@ -632,4 +632,14 @@ public function isWithoutErrorHandler(): self
             ),
         );
     }
+
+    public function isRetry(): self
+    {
+        return new self(
+            ...array_filter(
+                $this->metadata,
+                static fn (Metadata $metadata): bool => $metadata->isRetry(),
+            ),
+        );
+    }
 }

src/Metadata/Parser/AttributeParser.php

@@ -67,6 +67,7 @@
 use PHPUnit\Framework\Attributes\RequiresPhpunit;
 use PHPUnit\Framework\Attributes\RequiresPhpunitExtension;
 use PHPUnit\Framework\Attributes\RequiresSetting;
+use PHPUnit\Framework\Attributes\Retry;
 use PHPUnit\Framework\Attributes\RunClassInSeparateProcess;
 use PHPUnit\Framework\Attributes\RunInSeparateProcess;
 use PHPUnit\Framework\Attributes\RunTestsInSeparateProcesses;
@@ -845,6 +846,13 @@ public function forMethod(string $className, string $methodName): MetadataCollec
 
                     $result[] = Metadata::withoutErrorHandler();
 
+                    break;
+
+                case Retry::class:
+                    assert($attributeInstance instanceof Retry);
+
+                    $result[] = Metadata::retry($attributeInstance->maxRetries(), $attributeInstance->delay(), $attributeInstance->retryOn());
+
                     break;
             }
         }

src/Metadata/Retry.php

@@ -0,0 +1,61 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\Metadata;
+
+/**
+ * @immutable
+ *
+ * @no-named-arguments Parameter names are not covered by the backward compatibility promise for PHPUnit
+ */
+final readonly class Retry extends Metadata
+{
+    private int $maxRetries;
+    private int $delay;
+
+    /**
+     * @var ?non-empty-string
+     */
+    private ?string $retryOn;
+
+    /**
+     * @param ?non-empty-string $retryOn
+     */
+    public function __construct(int $level, int $maxRetries, int $delay, ?string $retryOn)
+    {
+        parent::__construct($level);
+
+        $this->maxRetries = $maxRetries;
+        $this->delay      = $delay;
+        $this->retryOn    = $retryOn;
+    }
+
+    public function isRetry(): bool
+    {
+        return true;
+    }
+
+    public function maxRetries(): int
+    {
+        return $this->maxRetries;
+    }
+
+    public function delay(): int
+    {
+        return $this->delay;
+    }
+
+    /**
+     * @return ?non-empty-string
+     */
+    public function retryOn(): ?string
+    {
+        return $this->retryOn;
+    }
+}

tests/_files/Metadata/Attribute/tests/RetryTest.php

@@ -0,0 +1,24 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\TestFixture\Metadata\Attribute;
+
+use LogicException;
+use PHPUnit\Framework\Attributes\Retry;
+use PHPUnit\Framework\TestCase;
+
+final class RetryTest extends TestCase
+{
+    #[Retry(1)]
+    #[Retry(2, 1)]
+    #[Retry(3, 0, LogicException::class)]
+    public function testOne(): void
+    {
+    }
+}

tests/unit/Framework/Attributes/RetryTest.php

@@ -0,0 +1,108 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\Framework\Attributes;
+
+use DateTime;
+use Exception;
+use LogicException;
+use PHPUnit\Framework\TestCase;
+
+final class RetryTest extends TestCase
+{
+    private static int $retryNumber = 0;
+    private static ?DateTime $start = null;
+
+    protected function setUp(): void
+    {
+        self::$retryNumber = 0;
+        self::$start       = new DateTime;
+    }
+
+    #[Retry(3)]
+    public function testRetriesUntilMaxAttemptsThenSucceeds(): void
+    {
+        if (self::$retryNumber < 3) {
+            self::$retryNumber++;
+
+            throw new Exception;
+        }
+
+        $this->assertSame(3, self::$retryNumber);
+    }
+
+    #[Retry(1)]
+    public function testSingleRetryThenThrowsExpectedException(): void
+    {
+        if (self::$retryNumber < 1) {
+            self::$retryNumber++;
+
+            throw new Exception;
+        }
+
+        $this->expectException(Exception::class);
+        $this->expectExceptionMessage('test exception two');
+        $this->assertSame(1, self::$retryNumber);
+
+        throw new Exception('test exception two');
+    }
+
+    #[Retry(2, 0, LogicException::class)]
+    public function testRetryWithUnmatchedExceptionTypeFailsImmediately(): void
+    {
+        $this->expectException(Exception::class);
+        $this->expectExceptionMessage('test exception');
+        $this->assertSame(0, self::$retryNumber);
+        self::$retryNumber++;
+
+        throw new Exception('test exception');
+    }
+
+    #[Retry(2, 0, LogicException::class)]
+    #[Retry(2)]
+    public function testMultipleRetryAttributesFallBackToDefaultRetry(): void
+    {
+        if (self::$retryNumber < 2) {
+            self::$retryNumber++;
+
+            throw new Exception;
+        }
+
+        $this->assertSame(2, self::$retryNumber);
+    }
+
+    #[Retry(5, 0, LogicException::class)]
+    public function testRetriesUntilLogicExceptionStopsThrowing(): void
+    {
+        if (self::$retryNumber < 5) {
+            self::$retryNumber++;
+
+            throw new LogicException;
+        }
+
+        $this->assertSame(5, self::$retryNumber);
+    }
+
+    #[Retry(1, 2)]
+    public function testRetryDelaysExecutionBySpecifiedSeconds(): void
+    {
+        $end = new DateTime;
+
+        if (self::$retryNumber < 1) {
+            self::$retryNumber++;
+
+            throw new Exception;
+        }
+
+        $diffInSeconds = $end->getTimestamp() - self::$start->getTimestamp();
+
+        $this->assertSame(1, self::$retryNumber);
+        $this->assertSame(2, $diffInSeconds);
+    }
+}