fix: split marker to ui_marker and mod_input_marker for test cases (#261)

UI tests may be slow to execute so we need to explore adding the ability
to use markers that can speed up test execution by parallelizing UI
tests. This is currently implemented only for modinput tests
([configuration example can be found
here](https://github.com/splunk/splunk-add-on-for-amazon-web-services/blob/84f5ec10221dc2d660c824c7b908dce58eea9cfa/.github/workflows/build-test-release.yml#L32-L33)).

Example workflow execution:
[https://github.com/splunk/splunk-add-on-for-amazon-web-services/pull/1220](https://github.com/splunk/splunk-add-on-for-amazon-web-services/pull/1220
)
Updated test runs: 
-
https://github.com/splunk/splunk-add-on-for-amazon-web-services/actions/runs/9254951314
-
https://github.com/splunk/test-addonfactory-repo/actions/runs/9254997916

---------

Co-authored-by: Artem Rys <rysartem@gmail.com>
Co-authored-by: kdoroszko-splunk <kdoroszko@splunk.com>
Co-authored-by: kdoroszko-splunk <147069837+kdoroszko-splunk@users.noreply.github.com>

Author: 3 people

.github/workflows/reusable-build-test-release.yml

@@ -4,7 +4,13 @@ on:
     inputs:
       marker:
         required: false
-        description: 'Parallel run marker'
+        description: 'Parallel run mod_input marker'
+        type: string
+        default: >-
+          [""]
+      ui_marker:
+        required: false
+        description: 'Parallel run ui marker'
         type: string
         default: >-
           [""]
@@ -963,7 +969,7 @@ jobs:
         continue-on-error: true
         env:
           ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }}
-        uses: splunk/wfe-test-runner-action@v4.0
+        uses: splunk/wfe-test-runner-action@v5.0
         with:
           splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}
           test-type: ${{ env.TEST_TYPE }}
@@ -1195,7 +1201,7 @@ jobs:
         continue-on-error: true
         env:
           ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }}
-        uses: splunk/wfe-test-runner-action@v4.0
+        uses: splunk/wfe-test-runner-action@v5.0
         with:
           splunk: ${{ matrix.splunk.version }}
           test-type: ${{ env.TEST_TYPE }}
@@ -1347,7 +1353,7 @@ jobs:
         splunk: ${{ fromJson(needs.meta.outputs.matrix_combinedSplunkversion) }}
         browser: [ "chrome" ]
         vendor-version: ${{ fromJson(needs.meta.outputs.matrix_supportedUIVendors) }}
-
+        marker: ${{ fromJson(inputs.ui_marker) }}
     container:
       image: ghcr.io/splunk/workflow-engine-base:4.0.0
     env:
@@ -1359,6 +1365,7 @@ jobs:
       SPLUNK_VERSION_BASE: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}
       TEST_TYPE: "ui"
       TEST_ARGS: "--browser ${{ matrix.browser }}"
+      TEST_BROWSER: ${{ matrix.browser }}
     permissions:
       actions: read
       deployments: read
@@ -1402,17 +1409,29 @@ jobs:
           JOB_NAME=${JOB_NAME//[_.:]/-}
           JOB_NAME=$(echo "$JOB_NAME" | tr '[:upper:]' '[:lower:]')
           echo "job-name=$JOB_NAME" >> "$GITHUB_OUTPUT"
+      - name: create test argument
+        id: create-test-arg
+        shell: bash
+        run: |
+          TEST_ARG_M=""
+          EMPTY_MARKER="[]"
+
+          if [[ "${{ inputs.ui_marker }}" != "$EMPTY_MARKER" ]]; then
+            TEST_ARG_M="-m"
+          fi
+
+          echo "test-arg=$TEST_ARG_M" >> "$GITHUB_OUTPUT"
       - name: run-tests
         id: run-tests
         timeout-minutes: 340
         continue-on-error: true
         env:
           ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }}
-        uses: splunk/wfe-test-runner-action@v4.0
+        uses: splunk/wfe-test-runner-action@v5.0
         with:
           splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}
           test-type: ${{ env.TEST_TYPE }}
-          test-args: ${{ env.TEST_ARGS }}
+          test-args: ${{ env.TEST_ARGS }} ${{ steps.create-test-arg.outputs.test-arg }} ${{ matrix.marker }}
           job-name: ${{ steps.create-job-name.outputs.job-name }}
           labels: ${{ needs.setup.outputs.labels }}
           workflow-tmpl-name: ${{ needs.setup.outputs.argo-workflow-tmpl-name }}
@@ -1422,6 +1441,7 @@ jobs:
           vendor-version: ${{ matrix.vendor-version.image }}
           sc4s-version: "No"
           k8s-manifests-branch: ${{ needs.setup.outputs.k8s-manifests-branch }}
+          test-browser: ${{ env.TEST_BROWSER }}
       - name: Read secrets from AWS Secrets Manager again into environment variables in case credential rotation
         id: update-argo-token
         if: ${{ !cancelled() }}
@@ -1625,21 +1645,21 @@ jobs:
         id: create-test-arg
         shell: bash
         run: |
-          export comparing_variable="[]"
-          if [ "${{ inputs.marker }}" == "$comparing_variable" ]
-          then
-            TEST_ARG_M=""
-          else
+          TEST_ARG_M=""
+          EMPTY_MARKER="[]"
+
+          if [[ "${{ inputs.marker }}" != "$EMPTY_MARKER" ]]; then
             TEST_ARG_M="-m"
           fi
+      
           echo "test-arg=$TEST_ARG_M" >> "$GITHUB_OUTPUT"
       - name: run-tests
         id: run-tests
         timeout-minutes: 340
         continue-on-error: true
         env:
           ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }}
-        uses: splunk/wfe-test-runner-action@v4.0
+        uses: splunk/wfe-test-runner-action@v5.0
         with:
           splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}
           test-type: ${{ env.TEST_TYPE }}
@@ -1868,7 +1888,7 @@ jobs:
         continue-on-error: true
         env:
           ARGO_TOKEN: ${{ steps.get-argo-token.outputs.argo-token }}
-        uses: splunk/wfe-test-runner-action@v4.0
+        uses: splunk/wfe-test-runner-action@v5.0
         with:
           splunk: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}
           test-type: ${{ env.TEST_TYPE }}

README.md

@@ -486,6 +486,9 @@ cim-compliance-report
 **Description**
 
 - This stage does the setup for executing UI tests and reports the results
+- It is possible to parallelize UI tests execution by using pytest markers. 
+  To do so, one must specify `ui_marker` parameter in buid-test-release.yml as in [example](https://github.com/splunk/splunk-add-on-for-amazon-web-services/blob/925fd189737507dd91cc5275c59a8b390550411c/.github/workflows/build-test-release.yml#L35).
+  Markers must be created prior and each test case must be marked (check [run-modinput-tests](#run-modinput-tests), and this [PR](https://github.com/splunk/splunk-add-on-for-amazon-web-services/pull/1237))
 
 **Action used:** 
 - No action used
@@ -525,6 +528,10 @@ Junit XML file
 **Description**
 
 - This stage does the setup for executing Modinput tests and reports the results
+- It is possible to parallelize Modinput tests execution by using pytest markers. 
+  To do so, one must specify `marker` parameter in buid-test-release.yml as in [example](https://github.com/splunk/splunk-add-on-for-amazon-web-services/blob/603f37ee24565f23104c0297e55a0c72480f34c9/.github/workflows/build-test-release.yml#L33).
+  Markers must be created prior and each test case must be marked (check the following references: [ref1](https://github.com/splunk/splunk-add-on-for-amazon-web-services/blob/main/tests/modinput_functional/README-test.md), 
+[ref2](https://github.com/splunk/splunk-add-on-for-amazon-web-services/blob/main/tests/modinput_functional/centaurs/tacommon/test_ta_mark.py), [ref3](https://github.com/splunk/splunk-add-on-for-amazon-web-services/blob/603f37ee24565f23104c0297e55a0c72480f34c9/tests/modinput_functional/data_collection/aws_s3/test_aws_s3_line_content.py#L20))
 
 **Action used:** 
 - No action