twister: harness_config extended by test retries in testcase.yaml

Extend Twister to allow defining repeat counts
for specific scenarios by harness_config.

Signed-off-by: Mateusz Junkier <mateusz.junkier@intel.com>

Author: majunkier

doc/develop/test/twister.rst

@@ -604,6 +604,18 @@ harness_config: <harness configuration options>
         Only one fixture can be defined per test scenario and the fixture name has to
         be unique across all tests in the test suite.
 
+    ztest_suite_repeat: <int> (default 1)
+        This parameter specifies the number of times the entire test suite should be repeated.
+
+    ztest_test_repeat: <int> (default 1)
+        This parameter specifies the number of times each individual test within the test suite
+        should be repeated.
+
+    ztest_test_shuffle: <True|False> (default False)
+        This parameter indicates whether the order of the tests within the test suite should
+        be shuffled. When set to ``true``, the tests will be executed in a random order.
+
+
 
     The following is an example yaml file with robot harness_config options.
 

scripts/pylib/twister/twisterlib/testinstance.py

@@ -66,9 +66,11 @@ def __init__(self, testsuite, platform, toolchain, outdir):
         self.build_time = 0
         self.retries = 0
         self.toolchain = toolchain
-
         self.name = os.path.join(platform.name, toolchain, testsuite.name)
         self.dut = None
+        self.suite_repeat = None
+        self.test_repeat = None
+        self.test_shuffle = None
 
         if testsuite.detailed_test_id:
             self.build_dir = os.path.join(
@@ -372,6 +374,23 @@ def create_overlay(
 
             content = "\n".join(new_config_list)
 
+
+        if self.testsuite.harness_config:
+            self.suite_repeat = self.testsuite.harness_config.get('ztest_suite_repeat', None)
+            self.test_repeat = self.testsuite.harness_config.get('ztest_test_repeat', None)
+            self.test_shuffle = self.testsuite.harness_config.get('ztest_test_shuffle', False)
+
+
+        # Use suite_repeat and test_repeat values
+        if self.suite_repeat or self.test_repeat or self.test_shuffle:
+            content +="\nCONFIG_ZTEST_REPEAT=y"
+            if self.suite_repeat:
+                content += f"\nCONFIG_ZTEST_SUITE_REPEAT_COUNT={self.suite_repeat}"
+            if self.test_repeat:
+                content += f"\nCONFIG_ZTEST_TEST_REPEAT_COUNT={self.test_repeat}"
+            if self.test_shuffle:
+                content +="\nCONFIG_ZTEST_SHUFFLE=y"
+
         if enable_coverage:
             for cp in coverage_platform:
                 if cp in platform.aliases:

scripts/schemas/twister/testsuite-schema.yaml

@@ -186,6 +186,15 @@ schema;scenario-schema:
         "bsim_exe_name":
           type: str
           required: false
+        "ztest_suite_repeat":
+          type: int
+          required: false
+        "ztest_test_repeat":
+          type: int
+          required: false
+        "ztest_test_shuffle":
+          type: bool
+          required: false
     "min_ram":
       type: int
       required: false

scripts/tests/twister/test_testinstance.py

@@ -651,3 +651,38 @@ def test_testinstance_get_buildlog_file(tmp_path, testinstance, create_build_log
 
     if expected_error is None:
         assert res == str(build_log)
+
+
+TESTDATA_9 = [
+    (
+        {'ztest_suite_repeat': 5, 'ztest_test_repeat': 10, 'ztest_test_shuffle': True},
+        '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SUITE_REPEAT_COUNT=5\nCONFIG_ZTEST_TEST_REPEAT_COUNT=10\nCONFIG_ZTEST_SHUFFLE=y'
+    ),
+    (
+        {'ztest_suite_repeat': 3},
+        '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SUITE_REPEAT_COUNT=3'
+    ),
+    (
+        {'ztest_test_repeat': 7},
+        '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_TEST_REPEAT_COUNT=7'
+    ),
+    (
+        {'ztest_test_shuffle': True},
+        '\nCONFIG_ZTEST_REPEAT=y\nCONFIG_ZTEST_SHUFFLE=y'
+    ),
+    (
+        {},
+        ''
+    ),
+]
+
+@pytest.mark.parametrize('harness_config, expected_content', TESTDATA_9)
+def test_create_overlay_with_harness_config(class_testplan, all_testsuites_dict, platforms_list, harness_config, expected_content):
+    testsuite_path = 'scripts/tests/twister/test_data/testsuites/samples/test_app/sample_test.app'
+    class_testplan.testsuites = all_testsuites_dict
+    testsuite = class_testplan.testsuites.get(testsuite_path)
+    testsuite.harness_config = harness_config
+    class_testplan.platforms = platforms_list
+    platform = class_testplan.get_platform("demo_board_2")
+    testinstance = TestInstance(testsuite, platform,'zephyr', class_testplan.env.outdir)
+    assert testinstance.create_overlay(platform) == expected_content

tests/ztest/repeat/CMakeLists.txt

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(repeat)
+
+target_sources(app PRIVATE src/main.c)

tests/ztest/repeat/prj.conf

@@ -0,0 +1,3 @@
+CONFIG_ZTEST=y
+CONFIG_ZTEST_SUITE_REPEAT_COUNT=3
+CONFIG_ZTEST_TEST_REPEAT_COUNT=2

tests/ztest/repeat/src/main.c

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2024 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#include <zephyr/ztest.h>
+
+extern struct ztest_suite_stats UTIL_CAT(z_ztest_suite_node_stats_, testsuite);
+struct ztest_suite_stats *suite_stats = &UTIL_CAT(z_ztest_suite_node_stats_, testsuite);
+extern struct ztest_unit_test_stats z_ztest_unit_test_stats_testsuite_test_repeating;
+struct ztest_unit_test_stats *case_stats =
+	&z_ztest_unit_test_stats_testsuite_test_repeating;
+
+static struct test_data {
+	int suite_run;
+	int case_run;
+} expected = {0, 0};
+
+ZTEST(testsuite, test_repeating)
+{
+	expected.case_run++;
+
+	if (case_stats->run_count > 0) {
+		zassert_true(case_stats->run_count == expected.case_run);
+	}
+
+	if (suite_stats->run_count > 0) {
+		zassert_true(suite_stats->run_count + 1 == expected.suite_run);
+	}
+}
+
+static void *repeat_setup(void)
+{
+	expected.suite_run++;
+	return NULL;
+}
+
+ZTEST_SUITE(testsuite, NULL, repeat_setup, NULL, NULL, NULL);

tests/ztest/repeat/testcase.yaml

@@ -0,0 +1,30 @@
+tests:
+  testing.ztest.repeat_suite_regex:
+    platform_allow: qemu_x86
+    harness: console
+    harness_config:
+      ztest_suite_repeat: 3
+      ztest_test_repeat: 2
+      type: multi_line
+      regex:
+        - "Running TESTSUITE testsuite"
+        - "Running TESTSUITE testsuite"
+        - "Running TESTSUITE testsuite"
+
+  testing.ztest.repeat_testcase_regex:
+    platform_allow: qemu_x86
+    harness: console
+    harness_config:
+      ztest_suite_repeat: 1
+      ztest_test_repeat: 5
+      type: multi_line
+      regex:
+        - "START - test_repeating"
+        - "START - test_repeating"
+        - "START - test_repeating"
+        - "START - test_repeating"
+        - "START - test_repeating"
+
+  testing.ztest.repeat_regex:
+    platform_allow: qemu_x86
+    harness: ztest