diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/rules/Retry.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/rules/Retry.kt new file mode 100644 index 00000000000..1eecacd1413 --- /dev/null +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/rules/Retry.kt @@ -0,0 +1,10 @@ +package com.woocommerce.android.e2e.rules + +/** + * Annotation used to denote you want to retry a UI test function. + * + * @property numberOfTimes the number of times you want to retry the function, with a default of 1. + */ +@Retention(AnnotationRetention.RUNTIME) +@Target(AnnotationTarget.FUNCTION) +annotation class Retry(val numberOfTimes: Int = 1) diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/rules/RetryTestRule.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/rules/RetryTestRule.kt new file mode 100644 index 00000000000..a234097b715 --- /dev/null +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/rules/RetryTestRule.kt @@ -0,0 +1,42 @@ +package com.woocommerce.android.e2e.rules + +import android.util.Log +import org.junit.rules.TestRule +import org.junit.runner.Description +import org.junit.runners.model.Statement + +const val TAG = "RetryTestRule" + +/** + * Custom rule used to retry running a test if a problem occurs. + * Credit: notandyvee + * https://github.com/wordpress-mobile/WordPress-Android/pull/20517 + */ +class RetryTestRule : TestRule { + override fun apply(base: Statement?, description: Description?): Statement { + return object : Statement() { + override fun evaluate() { + // We only retry functions that are annotated with @Retry. + val retry = description?.getAnnotation(Retry::class.java) + if (retry != null) { + var lastThrown: Throwable? = null + for (i in 0..retry.numberOfTimes) { + try { + base?.evaluate() + return + } catch (t: Throwable) { + Log.e(TAG, "Test failed to run due to problem on run $i", t) + lastThrown = t + } + } + Log.e(TAG, "Could not pass test.") + if (lastThrown != null) { + throw lastThrown + } + } + // If test function does not have @Retry, run as normal. + base?.evaluate() + } + } + } +} diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/screens/orders/SingleOrderScreen.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/screens/orders/SingleOrderScreen.kt index c3e6aa97b66..47b85059546 100644 --- a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/screens/orders/SingleOrderScreen.kt +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/screens/orders/SingleOrderScreen.kt @@ -14,9 +14,7 @@ import com.woocommerce.android.e2e.helpers.util.ProductData import com.woocommerce.android.e2e.helpers.util.Screen import org.hamcrest.Matchers -class SingleOrderScreen : Screen { - constructor() : super(R.id.toolbar) - +class SingleOrderScreen : Screen(R.id.toolbar) { fun goBackToOrdersScreen(): OrderListScreen { if (isElementDisplayed(R.id.orderDetail_container)) { pressBack() diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersRealAPI.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersRealAPI.kt index a28a88ad809..ec06114d39b 100644 --- a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersRealAPI.kt +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersRealAPI.kt @@ -8,6 +8,8 @@ import com.woocommerce.android.BuildConfig import com.woocommerce.android.e2e.helpers.InitializationRule import com.woocommerce.android.e2e.helpers.TestBase import com.woocommerce.android.e2e.helpers.useMockedAPI +import com.woocommerce.android.e2e.rules.Retry +import com.woocommerce.android.e2e.rules.RetryTestRule import com.woocommerce.android.e2e.screens.TabNavComponent import com.woocommerce.android.e2e.screens.login.WelcomeScreen import com.woocommerce.android.e2e.screens.orders.OrderListScreen @@ -37,6 +39,9 @@ class OrdersRealAPI : TestBase() { @get:Rule(order = 3) var activityRule = ActivityTestRule(LoginActivity::class.java) + @get:Rule(order = 4) + var retryTestRule = RetryTestRule() + companion object { @BeforeClass @JvmStatic @@ -73,6 +78,7 @@ class OrdersRealAPI : TestBase() { .logoutIfNeeded(composeTestRule) } + @Retry(numberOfTimes = 1) @Test fun e2eRealApiOrdersFilter() { OrderListScreen() @@ -91,6 +97,7 @@ class OrdersRealAPI : TestBase() { .assertOrdersCount(2) } + @Retry(numberOfTimes = 1) @Test fun e2eRealApiOrdersSearch() { OrderListScreen() @@ -115,6 +122,7 @@ class OrdersRealAPI : TestBase() { .assertOrdersCount(2) } + @Retry(numberOfTimes = 1) @Test @Ignore fun e2eRealApiOrderDetails() { diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersUITest.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersUITest.kt index 5d20c66cabc..80f221c6580 100644 --- a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersUITest.kt +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/OrdersUITest.kt @@ -10,6 +10,8 @@ import com.woocommerce.android.e2e.helpers.TestBase import com.woocommerce.android.e2e.helpers.util.MocksReader import com.woocommerce.android.e2e.helpers.util.OrderData import com.woocommerce.android.e2e.helpers.util.iterator +import com.woocommerce.android.e2e.rules.Retry +import com.woocommerce.android.e2e.rules.RetryTestRule import com.woocommerce.android.e2e.screens.TabNavComponent import com.woocommerce.android.e2e.screens.login.WelcomeScreen import com.woocommerce.android.e2e.screens.orders.OrderListScreen @@ -36,6 +38,9 @@ class OrdersUITest : TestBase() { @get:Rule(order = 3) var activityRule = ActivityTestRule(LoginActivity::class.java) + @get:Rule(order = 4) + var retryTestRule = RetryTestRule() + @Before fun setUp() { WelcomeScreen @@ -48,6 +53,7 @@ class OrdersUITest : TestBase() { TabNavComponent().gotoOrdersScreen() } + @Retry(numberOfTimes = 1) @Test @Ignore fun e2eCreateOrderTest() { diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsRealAPI.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsRealAPI.kt index c0dcf78ae36..9757a3e351b 100644 --- a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsRealAPI.kt +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsRealAPI.kt @@ -8,6 +8,8 @@ import com.woocommerce.android.BuildConfig import com.woocommerce.android.e2e.helpers.InitializationRule import com.woocommerce.android.e2e.helpers.TestBase import com.woocommerce.android.e2e.helpers.useMockedAPI +import com.woocommerce.android.e2e.rules.Retry +import com.woocommerce.android.e2e.rules.RetryTestRule import com.woocommerce.android.e2e.screens.TabNavComponent import com.woocommerce.android.e2e.screens.login.WelcomeScreen import com.woocommerce.android.e2e.screens.products.ProductListScreen @@ -35,6 +37,9 @@ class ProductsRealAPI : TestBase() { @get:Rule(order = 3) var activityRule = ActivityTestRule(LoginActivity::class.java) + @get:Rule(order = 4) + var retryTestRule = RetryTestRule() + companion object { @BeforeClass @JvmStatic @@ -71,6 +76,7 @@ class ProductsRealAPI : TestBase() { .logoutIfNeeded(composeTestRule) } + @Retry(numberOfTimes = 1) @Test fun e2eRealApiProductsSearchUsual() { ProductListScreen() @@ -103,6 +109,7 @@ class ProductsRealAPI : TestBase() { .assertProductsCount(2) } + @Retry(numberOfTimes = 1) @Test fun e2eRealApiProductsSearchBySKU() { ProductListScreen() @@ -130,6 +137,7 @@ class ProductsRealAPI : TestBase() { .leaveSearchMode() } + @Retry(numberOfTimes = 1) @Test fun e2eRealApiProductsFilter() { ProductListScreen() @@ -153,6 +161,7 @@ class ProductsRealAPI : TestBase() { .assertProductsCount(0) } + @Retry(numberOfTimes = 1) @Test fun e2eRealApiProductsSort() { ProductListScreen() diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsUITest.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsUITest.kt index bef96e74fc9..0c2e7d21d34 100644 --- a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsUITest.kt +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ProductsUITest.kt @@ -9,6 +9,8 @@ import com.woocommerce.android.e2e.helpers.TestBase import com.woocommerce.android.e2e.helpers.util.MocksReader import com.woocommerce.android.e2e.helpers.util.ProductData import com.woocommerce.android.e2e.helpers.util.iterator +import com.woocommerce.android.e2e.rules.Retry +import com.woocommerce.android.e2e.rules.RetryTestRule import com.woocommerce.android.e2e.screens.TabNavComponent import com.woocommerce.android.e2e.screens.login.WelcomeScreen import com.woocommerce.android.e2e.screens.products.ProductListScreen @@ -31,6 +33,9 @@ class ProductsUITest : TestBase() { @get:Rule(order = 2) var activityRule = ActivityTestRule(LoginActivity::class.java) + @get:Rule(order = 3) + var retryTestRule = RetryTestRule() + @Before fun setUp() { WelcomeScreen @@ -43,6 +48,7 @@ class ProductsUITest : TestBase() { TabNavComponent().gotoProductsScreen() } + @Retry(numberOfTimes = 1) @Test fun e2eProductListShowsAllProducts() { val productsJSONArray = MocksReader().readAllProductsToArray() diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ReviewsUITest.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ReviewsUITest.kt index 9ce010e5d10..c97f94d43b7 100644 --- a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ReviewsUITest.kt +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/ReviewsUITest.kt @@ -10,6 +10,8 @@ import com.woocommerce.android.e2e.helpers.TestBase import com.woocommerce.android.e2e.helpers.util.MocksReader import com.woocommerce.android.e2e.helpers.util.ReviewData import com.woocommerce.android.e2e.helpers.util.iterator +import com.woocommerce.android.e2e.rules.Retry +import com.woocommerce.android.e2e.rules.RetryTestRule import com.woocommerce.android.e2e.screens.TabNavComponent import com.woocommerce.android.e2e.screens.login.WelcomeScreen import com.woocommerce.android.e2e.screens.reviews.ReviewsListScreen @@ -34,6 +36,9 @@ class ReviewsUITest : TestBase() { @get:Rule(order = 3) var activityRule = ActivityTestRule(LoginActivity::class.java) + @get:Rule(order = 4) + var retryTestRule = RetryTestRule() + @Before fun setUp() { WelcomeScreen @@ -48,6 +53,7 @@ class ReviewsUITest : TestBase() { .openReviewsListScreen(composeTestRule) } + @Retry(numberOfTimes = 1) @Test fun e2eReviewListShowsAllReviews() { val reviewsJSONArray = MocksReader().readAllReviewsToArray() diff --git a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/StatsUITest.kt b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/StatsUITest.kt index 8459634409d..f395d68c46a 100644 --- a/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/StatsUITest.kt +++ b/WooCommerce/src/androidTest/kotlin/com/woocommerce/android/e2e/tests/ui/StatsUITest.kt @@ -9,6 +9,8 @@ import com.woocommerce.android.e2e.helpers.InitializationRule import com.woocommerce.android.e2e.helpers.TestBase import com.woocommerce.android.e2e.helpers.util.MocksReader import com.woocommerce.android.e2e.helpers.util.StatsSummaryData +import com.woocommerce.android.e2e.rules.Retry +import com.woocommerce.android.e2e.rules.RetryTestRule import com.woocommerce.android.e2e.screens.TabNavComponent import com.woocommerce.android.e2e.screens.login.WelcomeScreen import com.woocommerce.android.e2e.screens.mystore.MyStoreScreen @@ -16,7 +18,6 @@ import com.woocommerce.android.ui.login.LoginActivity import dagger.hilt.android.testing.HiltAndroidRule import dagger.hilt.android.testing.HiltAndroidTest import org.junit.Before -import org.junit.Ignore import org.junit.Rule import org.junit.Test @@ -34,6 +35,9 @@ class StatsUITest : TestBase() { @get:Rule(order = 3) var activityRule = ActivityTestRule(LoginActivity::class.java) + @get:Rule(order = 4) + var retryTestRule = RetryTestRule() + @Before fun setUp() { WelcomeScreen @@ -71,6 +75,7 @@ class StatsUITest : TestBase() { visitors = "12000", ) + @Retry(numberOfTimes = 1) @Test fun e2eStatsSummary() { MyStoreScreen() @@ -82,6 +87,7 @@ class StatsUITest : TestBase() { .assertStatsSummary(yearStats) } + @Retry(numberOfTimes = 1) @Test fun e2eStatsTopPerformers() { val topPerformersJSONArray = MocksReader().readStatsTopPerformersToArray() @@ -91,7 +97,7 @@ class StatsUITest : TestBase() { .assertTopPerformers(topPerformersJSONArray) } - @Ignore + @Retry(numberOfTimes = 1) @Test fun e2eStatsTapChart() { MyStoreScreen()