[UI Tests] Added a rerun parameter

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/rules/Retry.kt

@@ -0,0 +1,10 @@
+package com.woocommerce.android.e2e.rules
+
+/**
+ * Annotation used to denote you want to retry a UI test function.
+ *
+ * @property numberOfTimes the number of times you want to retry the function, with a default of 1.
+ */
+@Retention(AnnotationRetention.RUNTIME)
+@Target(AnnotationTarget.FUNCTION)
+annotation class Retry(val numberOfTimes: Int = 1)

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/rules/RetryTestRule.kt

@@ -0,0 +1,42 @@
+package com.woocommerce.android.e2e.rules
+
+import android.util.Log
+import org.junit.rules.TestRule
+import org.junit.runner.Description
+import org.junit.runners.model.Statement
+
+const val TAG = "RetryTestRule"
+
+/**
+ * Custom rule used to retry running a test if a problem occurs.
+ * Credit: notandyvee
+ * https://github.com/wordpress-mobile/WordPress-Android/pull/20517
+ */
+class RetryTestRule : TestRule {
+    override fun apply(base: Statement?, description: Description?): Statement {
+        return object : Statement() {
+            override fun evaluate() {
+                // We only retry functions that are annotated with @Retry.
+                val retry = description?.getAnnotation(Retry::class.java)
+                if (retry != null) {
+                    var lastThrown: Throwable? = null
+                    for (i in 0..retry.numberOfTimes) {
+                        try {
+                            base?.evaluate()
+                            return
+                        } catch (t: Throwable) {
+                            Log.e(TAG, "Test failed to run due to problem on run $i", t)
+                            lastThrown = t
+                        }
+                    }
+                    Log.e(TAG, "Could not pass test.")
+                    if (lastThrown != null) {
+                        throw lastThrown
+                    }
+                }
+                // If test function does not have @Retry, run as normal.
+                base?.evaluate()
+            }
+        }
+    }
+}

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/screens/orders/SingleOrderScreen.kt

@@ -14,9 +14,7 @@ import com.woocommerce.android.e2e.helpers.util.ProductData
 import com.woocommerce.android.e2e.helpers.util.Screen
 import org.hamcrest.Matchers
 
-class SingleOrderScreen : Screen {
-    constructor() : super(R.id.toolbar)
-
+class SingleOrderScreen : Screen(R.id.toolbar) {
     fun goBackToOrdersScreen(): OrderListScreen {
         if (isElementDisplayed(R.id.orderDetail_container)) {
             pressBack()

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersRealAPI.kt

@@ -8,6 +8,8 @@ import com.woocommerce.android.BuildConfig
 import com.woocommerce.android.e2e.helpers.InitializationRule
 import com.woocommerce.android.e2e.helpers.TestBase
 import com.woocommerce.android.e2e.helpers.useMockedAPI
+import com.woocommerce.android.e2e.rules.Retry
+import com.woocommerce.android.e2e.rules.RetryTestRule
 import com.woocommerce.android.e2e.screens.TabNavComponent
 import com.woocommerce.android.e2e.screens.login.WelcomeScreen
 import com.woocommerce.android.e2e.screens.orders.OrderListScreen
@@ -37,6 +39,9 @@ class OrdersRealAPI : TestBase() {
     @get:Rule(order = 3)
     var activityRule = ActivityTestRule(LoginActivity::class.java)
 
+    @get:Rule(order = 4)
+    var retryTestRule = RetryTestRule()
+
     companion object {
         @BeforeClass
         @JvmStatic
@@ -73,6 +78,7 @@ class OrdersRealAPI : TestBase() {
             .logoutIfNeeded(composeTestRule)
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eRealApiOrdersFilter() {
         OrderListScreen()
@@ -91,6 +97,7 @@ class OrdersRealAPI : TestBase() {
             .assertOrdersCount(2)
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eRealApiOrdersSearch() {
         OrderListScreen()
@@ -115,6 +122,7 @@ class OrdersRealAPI : TestBase() {
             .assertOrdersCount(2)
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     @Ignore
     fun e2eRealApiOrderDetails() {

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersUITest.kt

@@ -10,6 +10,8 @@ import com.woocommerce.android.e2e.helpers.TestBase
 import com.woocommerce.android.e2e.helpers.util.MocksReader
 import com.woocommerce.android.e2e.helpers.util.OrderData
 import com.woocommerce.android.e2e.helpers.util.iterator
+import com.woocommerce.android.e2e.rules.Retry
+import com.woocommerce.android.e2e.rules.RetryTestRule
 import com.woocommerce.android.e2e.screens.TabNavComponent
 import com.woocommerce.android.e2e.screens.login.WelcomeScreen
 import com.woocommerce.android.e2e.screens.orders.OrderListScreen
@@ -36,6 +38,9 @@ class OrdersUITest : TestBase() {
     @get:Rule(order = 3)
     var activityRule = ActivityTestRule(LoginActivity::class.java)
 
+    @get:Rule(order = 4)
+    var retryTestRule = RetryTestRule()
+
     @Before
     fun setUp() {
         WelcomeScreen
@@ -48,6 +53,7 @@ class OrdersUITest : TestBase() {
         TabNavComponent().gotoOrdersScreen()
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     @Ignore
     fun e2eCreateOrderTest() {

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsRealAPI.kt

@@ -8,6 +8,8 @@ import com.woocommerce.android.BuildConfig
 import com.woocommerce.android.e2e.helpers.InitializationRule
 import com.woocommerce.android.e2e.helpers.TestBase
 import com.woocommerce.android.e2e.helpers.useMockedAPI
+import com.woocommerce.android.e2e.rules.Retry
+import com.woocommerce.android.e2e.rules.RetryTestRule
 import com.woocommerce.android.e2e.screens.TabNavComponent
 import com.woocommerce.android.e2e.screens.login.WelcomeScreen
 import com.woocommerce.android.e2e.screens.products.ProductListScreen
@@ -35,6 +37,9 @@ class ProductsRealAPI : TestBase() {
     @get:Rule(order = 3)
     var activityRule = ActivityTestRule(LoginActivity::class.java)
 
+    @get:Rule(order = 4)
+    var retryTestRule = RetryTestRule()
+
     companion object {
         @BeforeClass
         @JvmStatic
@@ -71,6 +76,7 @@ class ProductsRealAPI : TestBase() {
             .logoutIfNeeded(composeTestRule)
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eRealApiProductsSearchUsual() {
         ProductListScreen()
@@ -103,6 +109,7 @@ class ProductsRealAPI : TestBase() {
             .assertProductsCount(2)
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eRealApiProductsSearchBySKU() {
         ProductListScreen()
@@ -130,6 +137,7 @@ class ProductsRealAPI : TestBase() {
             .leaveSearchMode()
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eRealApiProductsFilter() {
         ProductListScreen()
@@ -153,6 +161,7 @@ class ProductsRealAPI : TestBase() {
             .assertProductsCount(0)
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eRealApiProductsSort() {
         ProductListScreen()

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsUITest.kt

@@ -9,6 +9,8 @@ import com.woocommerce.android.e2e.helpers.TestBase
 import com.woocommerce.android.e2e.helpers.util.MocksReader
 import com.woocommerce.android.e2e.helpers.util.ProductData
 import com.woocommerce.android.e2e.helpers.util.iterator
+import com.woocommerce.android.e2e.rules.Retry
+import com.woocommerce.android.e2e.rules.RetryTestRule
 import com.woocommerce.android.e2e.screens.TabNavComponent
 import com.woocommerce.android.e2e.screens.login.WelcomeScreen
 import com.woocommerce.android.e2e.screens.products.ProductListScreen
@@ -31,6 +33,9 @@ class ProductsUITest : TestBase() {
     @get:Rule(order = 2)
     var activityRule = ActivityTestRule(LoginActivity::class.java)
 
+    @get:Rule(order = 3)
+    var retryTestRule = RetryTestRule()
+
     @Before
     fun setUp() {
         WelcomeScreen
@@ -43,6 +48,7 @@ class ProductsUITest : TestBase() {
         TabNavComponent().gotoProductsScreen()
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eProductListShowsAllProducts() {
         val productsJSONArray = MocksReader().readAllProductsToArray()

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ReviewsUITest.kt

@@ -10,6 +10,8 @@ import com.woocommerce.android.e2e.helpers.TestBase
 import com.woocommerce.android.e2e.helpers.util.MocksReader
 import com.woocommerce.android.e2e.helpers.util.ReviewData
 import com.woocommerce.android.e2e.helpers.util.iterator
+import com.woocommerce.android.e2e.rules.Retry
+import com.woocommerce.android.e2e.rules.RetryTestRule
 import com.woocommerce.android.e2e.screens.TabNavComponent
 import com.woocommerce.android.e2e.screens.login.WelcomeScreen
 import com.woocommerce.android.e2e.screens.reviews.ReviewsListScreen
@@ -34,6 +36,9 @@ class ReviewsUITest : TestBase() {
     @get:Rule(order = 3)
     var activityRule = ActivityTestRule(LoginActivity::class.java)
 
+    @get:Rule(order = 4)
+    var retryTestRule = RetryTestRule()
+
     @Before
     fun setUp() {
         WelcomeScreen
@@ -48,6 +53,7 @@ class ReviewsUITest : TestBase() {
             .openReviewsListScreen(composeTestRule)
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eReviewListShowsAllReviews() {
         val reviewsJSONArray = MocksReader().readAllReviewsToArray()

WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/StatsUITest.kt

@@ -9,14 +9,15 @@ import com.woocommerce.android.e2e.helpers.InitializationRule
 import com.woocommerce.android.e2e.helpers.TestBase
 import com.woocommerce.android.e2e.helpers.util.MocksReader
 import com.woocommerce.android.e2e.helpers.util.StatsSummaryData
+import com.woocommerce.android.e2e.rules.Retry
+import com.woocommerce.android.e2e.rules.RetryTestRule
 import com.woocommerce.android.e2e.screens.TabNavComponent
 import com.woocommerce.android.e2e.screens.login.WelcomeScreen
 import com.woocommerce.android.e2e.screens.mystore.MyStoreScreen
 import com.woocommerce.android.ui.login.LoginActivity
 import dagger.hilt.android.testing.HiltAndroidRule
 import dagger.hilt.android.testing.HiltAndroidTest
 import org.junit.Before
-import org.junit.Ignore
 import org.junit.Rule
 import org.junit.Test
 
@@ -34,6 +35,9 @@ class StatsUITest : TestBase() {
     @get:Rule(order = 3)
     var activityRule = ActivityTestRule(LoginActivity::class.java)
 
+    @get:Rule(order = 4)
+    var retryTestRule = RetryTestRule()
+
     @Before
     fun setUp() {
         WelcomeScreen
@@ -71,6 +75,7 @@ class StatsUITest : TestBase() {
         visitors = "12000",
     )
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eStatsSummary() {
         MyStoreScreen()
@@ -82,6 +87,7 @@ class StatsUITest : TestBase() {
             .assertStatsSummary(yearStats)
     }
 
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eStatsTopPerformers() {
         val topPerformersJSONArray = MocksReader().readStatsTopPerformersToArray()
@@ -91,7 +97,7 @@ class StatsUITest : TestBase() {
             .assertTopPerformers(topPerformersJSONArray)
     }
 
-    @Ignore
+    @Retry(numberOfTimes = 1)
     @Test
     fun e2eStatsTapChart() {
         MyStoreScreen()